Preparing to flash…
diff --git a/mooninstaller/install-orchestrator.js b/mooninstaller/install-orchestrator.js
index 6af2be9d3..6a343ad68 100644
--- a/mooninstaller/install-orchestrator.js
+++ b/mooninstaller/install-orchestrator.js
@@ -71,6 +71,16 @@ import {
encodeApplyOpFrames,
stateFrames,
} from "./improv-frame.js";
+import {
+ TEENSY_FAMILY,
+ imageProblem,
+ isTeensySerial,
+ rebootToBootloader,
+ rebootedSerialPort,
+ otherTeensyPorts,
+ requestHalfKay,
+ writeImage,
+} from "./teensy-writer.js";
// ---------------------------------------------------------------------------
// Manifest parser
@@ -445,6 +455,84 @@ export function flashPercent(sizes, idx, written, total) {
return pct < 0 ? 0 : pct > 100 ? 100 : pct;
}
+// Installs a Teensy 4.1 through its HalfKay bootloader (teensy-writer.js), from a running board's serial port or, with no port, from a board held in its bootloader by the program button.
+// The outcome is the address its boot log prints, and the catalog defaults go over the same serial port as the ESP32's do (APPLY_OP); there is no WiFi to provision.
+async function installTeensy({ manifest, port, device, applyDefaults, eraseBefore, trackProgress, onProgress, onLog, uiGrantBootloader, uiFindBoard, onSuccess }) {
+ if (!navigator.hid) throw new Error("WebHID is not available: install a Teensy from Chrome or Edge on a desktop computer");
+ if (!uiGrantBootloader) throw new Error("the installer cannot ask for the Teensy bootloader");
+ const log = (line) => { if (onLog) onLog(`[teensy] ${line}`); };
+
+ // The bootloader, picked in the browser's chooser.
+ // The chooser needs a recent click and does not remember the grant, so it opens straight from the Install click while that still counts (a few seconds), listing the bootloader as it appears.
+ // When the click has expired, or the user cancels, the page says what to do and a click of its own opens the chooser.
+ const bootloader = async (pressButton) => {
+ try {
+ const dev = await requestHalfKay();
+ if (dev) return dev;
+ log("no bootloader picked in the browser's list");
+ } catch (e) {
+ log(`the browser's list needs a fresh click (${e && e.message || e})`);
+ }
+ trackProgress("grant-bootloader", { pressButton });
+ log("waiting for the bootloader to be chosen");
+ return await uiGrantBootloader();
+ };
+
+ // A board held in its bootloader has no serial port: the bootloader is the target from the start.
+ let halfkay = port ? null : await bootloader(true);
+ // Before the reboot, while the installed board still answers on `port` (or on none), so its port after the reboot is the one not here.
+ const others = await otherTeensyPorts(port);
+
+ trackProgress("fetch-firmware");
+ log(`downloading ${manifest.parts[0].url}`);
+ const url = manifest.parts[0].url;
+ const res = await fetch(url);
+ if (!res.ok) throw new Error(`firmware fetch HTTP ${res.status}: ${url}`);
+ const image = new Uint8Array(await res.arrayBuffer());
+ const problem = imageProblem(image);
+ if (problem) throw new Error(problem);
+
+ log(`image ${image.length} bytes`);
+ if (!halfkay) {
+ trackProgress("reboot-bootloader");
+ log("asking the running board into its bootloader (134 baud)");
+ await rebootToBootloader(port);
+ halfkay = await bootloader(false);
+ }
+
+ trackProgress("flash", { pct: 0 });
+ await writeImage(halfkay, image, { onLog, eraseSettings: eraseBefore, onProgress: (pct) => onProgress("flash", { pct }) });
+
+ trackProgress("reboot");
+ let ip = "", mdns = "", defaultsApplied = false;
+ let serial = await rebootedSerialPort(15000, others);
+ if (!serial && uiFindBoard) {
+ // The board was installed from its bootloader, so the installer may not read its serial port yet: one click grants it.
+ log("no granted serial port for the board: asking for one");
+ trackProgress("find-board");
+ serial = await uiFindBoard();
+ }
+ log(serial ? "reading the board's address from its serial log" : "no serial port for the board: no address to show");
+ if (serial) {
+ trackProgress("read-address");
+ try {
+ await serial.open({ baudRate: 115200 });
+ ({ ip, mdns } = await readBootIp(serial, 20000));
+ defaultsApplied = await pushDefaultsOverSerial(serial, device, applyDefaults, trackProgress, onLog);
+ } catch (e) {
+ if (onLog) onLog(`[teensy] could not read the boot log (${e && e.message || e})`);
+ } finally {
+ try { await serial.close(); } catch (_) { /* ignore */ }
+ }
+ }
+ trackProgress("done");
+ if (ip) {
+ onSuccess({ url: `http://${ip}/`, mdns, device: device || "", applyDefaults, defaultsApplied, viaHttp: true, alreadyOnline: true });
+ } else {
+ onSuccess({ url: "", device: device || "", applyDefaults, defaultsApplied, ethOnlyNoLink: true });
+ }
+}
+
export const installer = {
/**
* Drive the full install flow: request port, flash via esptool-js,
@@ -532,13 +620,20 @@ export const installer = {
* When provided the orchestrator skips its own requestPort() prompt
* and uses this handle directly. Falls back to requestPort() when
* omitted/null OR when opening this handle fails (stale grant after
- * the device was unplugged and replugged).
+ * the device was unplugged and replugged). A Teensy manifest with no
+ * port means a board held in its bootloader by its program button.
+ * @param {() => Promise
} [opts.uiGrantBootloader]
+ * Teensy only: host shows a button whose click calls requestHalfKay(),
+ * and resolves once the user has picked the bootloader in the browser's list.
+ * @param {() => Promise} [opts.uiFindBoard]
+ * Teensy only: host shows a button whose click calls requestTeensySerial(),
+ * resolving with the board's port, or null when the user skips.
*/
async start({ manifestUrl, device, applyDefaults = true, txPower = null, eraseBefore = false,
ethOnly = false,
port: prePickedPort,
onProgress, uiWaitForCreds, uiWaitForIp, uiShowNeedsIpRetrying,
- uiWaitForPortRetry,
+ uiWaitForPortRetry, uiGrantBootloader, uiFindBoard,
onSuccess, onError, onLog }) {
if (!navigator.serial) {
onError("setup", new Error("Web Serial API not available — use Chrome, Edge, or Opera"));
@@ -595,6 +690,14 @@ export const installer = {
});
try {
+ // The manifest's chip family picks the writer, before anything touches the device.
+ const manifest = await fetchManifest(manifestUrl);
+ if (manifest.chipFamily === TEENSY_FAMILY) {
+ if (_detected) await releaseDetected();
+ await installTeensy({ manifest, port: prePickedPort, device, applyDefaults, eraseBefore, trackProgress,
+ onProgress, onLog, uiGrantBootloader, uiFindBoard, onSuccess });
+ return;
+ }
trackProgress("request-port");
// Use the pre-picked port when the host page provided one (the
// "USB Port" dropdown on the install page). Otherwise surface
@@ -715,7 +818,6 @@ export const installer = {
// Failing here (CDN issue, bad manifest) leaves the device
// intact — better than mid-flash failure with an erased chip.
trackProgress("fetch-firmware");
- const manifest = await fetchManifest(manifestUrl);
const fileArray = [];
for (const part of manifest.parts) {
const res = await fetch(part.url);
@@ -1144,6 +1246,8 @@ export const installer = {
await releaseDetected();
const port = prePickedPort || await navigator.serial.requestPort({});
+ // A Teensy names itself by its USB ids, and an ESP32 handshake would only confuse it.
+ if (isTeensySerial(port.getInfo())) return TEENSY_FAMILY;
// Trimmed espTerminal (same shape as start()'s) so chip-info lines reach
// the log panel; the single-char filter drops esptool's "." / "_" noise.
const espTerminal = onLog ? {
diff --git a/mooninstaller/install.js b/mooninstaller/install.js
index 67807f6a9..44413c0d8 100644
--- a/mooninstaller/install.js
+++ b/mooninstaller/install.js
@@ -18,6 +18,7 @@
import { installPicker } from "./install-picker.js";
import { myDevices } from "./devices.js";
import { installer, ESPTOOL_JS_VERSION } from "./install-orchestrator.js";
+import { TEENSY_FAMILY, requestHalfKay, requestTeensySerial } from "./teensy-writer.js";
// Device catalog + chip detection: mooninstaller only, kept out of the
// firmware-embedded install-picker.js and injected here via deviceSupport.
import * as deviceSupport from "./install-picker-devices.js";
@@ -104,6 +105,8 @@ document.addEventListener('DOMContentLoaded', () => {
const sections = {
connecting: document.getElementById("section-connecting"),
wrongPort: document.getElementById("section-wrong-port"),
+ grantBootloader: document.getElementById("section-grant-bootloader"),
+ findBoard: document.getElementById("section-find-board"),
flashing: document.getElementById("section-flashing"),
wifiForm: document.getElementById("section-wifi-form"),
provisioning: document.getElementById("section-provisioning"),
@@ -265,6 +268,27 @@ document.addEventListener('DOMContentLoaded', () => {
bar.style.width = `${pct}%`;
break;
}
+ case "reboot-bootloader":
+ showSection("connecting");
+ document.getElementById("connecting-detail").textContent =
+ "Asking the Teensy into its bootloader…";
+ break;
+ case "grant-bootloader":
+ showSection("grantBootloader");
+ document.getElementById("grant-bootloader-text").textContent = detail && detail.pressButton
+ ? "Press the program button on the Teensy now, then click below and pick \"Unknown device (16C0:0478)\", the bootloader, in the browser's list."
+ : "The Teensy is in its bootloader. Click below and pick \"Unknown device (16C0:0478)\", the bootloader, in the browser's list.";
+ document.getElementById("grant-bootloader-hint").hidden = true;
+ break;
+ case "find-board":
+ showSection("findBoard");
+ document.getElementById("find-board-hint").hidden = true;
+ break;
+ case "read-address":
+ showSection("connecting");
+ document.getElementById("connecting-detail").textContent =
+ "Waiting for the board's network address…";
+ break;
case "reboot":
showSection("connecting");
document.getElementById("connecting-detail").textContent =
@@ -429,6 +453,52 @@ document.addEventListener('DOMContentLoaded', () => {
});
}
+ // Resolves with the Teensy bootloader device the user picks in the browser's list.
+ // requestDevice needs a user gesture, so it runs inside the button's click; a cancel or an empty list keeps the step open with a hint.
+ function uiGrantBootloader() {
+ return new Promise((resolve) => {
+ const btn = document.getElementById("grant-bootloader");
+ const hint = document.getElementById("grant-bootloader-hint");
+ const onClick = async () => {
+ let dev = null;
+ try { dev = await requestHalfKay(); } catch (_) { /* canceled */ }
+ if (!dev) {
+ hint.textContent = "No bootloader chosen. Is the Teensy plugged in, and was its program button pressed? Then click again.";
+ hint.hidden = false;
+ return;
+ }
+ btn.removeEventListener("click", onClick);
+ resolve(dev);
+ };
+ btn.addEventListener("click", onClick);
+ });
+ }
+
+ // Resolves with the rebooted Teensy's serial port, picked in the browser's list, or null on Skip.
+ // requestPort needs a user gesture, so it runs inside the button's click; a cancel keeps the step open.
+ function uiFindBoard() {
+ return new Promise((resolve) => {
+ const btn = document.getElementById("find-board");
+ const skip = document.getElementById("find-board-skip");
+ const hint = document.getElementById("find-board-hint");
+ const done = (port) => {
+ btn.removeEventListener("click", onClick);
+ skip.removeEventListener("click", onSkip);
+ resolve(port);
+ };
+ const onClick = async () => {
+ let port = null;
+ try { port = await requestTeensySerial(); } catch (_) { /* canceled */ }
+ if (port) { done(port); return; }
+ hint.textContent = "No port chosen. Pick the Teensy's serial port in the list, or skip.";
+ hint.hidden = false;
+ };
+ const onSkip = () => done(null);
+ btn.addEventListener("click", onClick);
+ skip.addEventListener("click", onSkip);
+ });
+ }
+
// Symmetric with uiWaitForCreds. Normalization of the typed value lives
// in the orchestrator so the host page stays UI-only. The retry action
// is the orchestrator's signal to re-run Improv `initialize()`; while
@@ -605,6 +675,7 @@ document.addEventListener('DOMContentLoaded', () => {
// vocabulary build_esp32's TARGET_TO_FAMILY uses. Used as the chip fallback when no
// device is picked (OTA / generic flash). "" for an unrecognized key.
function firmwareToChip(firmware) {
+ if (/^teensy41/.test(String(firmware || ""))) return TEENSY_FAMILY;
const m = String(firmware || "").match(/^esp32(s31|s3|s2|p4|c\d+|h\d+)/);
return m ? (m[1] === "s31" ? "ESP32-S31" : "ESP32-" + m[1].toUpperCase()) : (firmware ? "ESP32" : "");
}
@@ -667,9 +738,13 @@ document.addEventListener('DOMContentLoaded', () => {
// differs. Kept beside pickedPort because the two are mutually exclusive states of the
// one "what am I installing onto" question the Port row asks.
let desktopMode = false; // this page is for flashing an ESP32; the desktop download is the alternative
+ // A Teensy held in its bootloader by its program button: no serial port, the bootloader itself is the target.
+ // Not a dropdown choice: the port picker offers it when no Teensy is listed, the one case that needs it.
+ let teensyBootMode = false;
const portSelect = document.getElementById("port-select");
const PICK_NEW = "__pick_new__"; // sentinel value for the "pick another" option
const DESKTOP = "__desktop__"; // sentinel value for the "this computer" option
+ const TEENSY_BOOT = "__teensy_boot__"; // sentinel value naming the bootloader mode while it is active
const IDLE = "__idle__"; // neutral placeholder: nothing chosen yet
// Name this computer the way the picker names its archive, or null on an OS we do not
@@ -697,12 +772,20 @@ document.addEventListener('DOMContentLoaded', () => {
// have to grant a serial port to discover it exists.
const desktopLabel = thisComputerLabel();
const addDesktopOpt = () => {
+ // Shown only while active, so the row names the target the hint below the port picker chose.
+ if (teensyBootMode) {
+ const teensyOpt = document.createElement("option");
+ teensyOpt.value = TEENSY_BOOT;
+ teensyOpt.textContent = "A Teensy in its bootloader";
+ portSelect.appendChild(teensyOpt);
+ }
if (!desktopLabel) return;
const opt = document.createElement("option");
opt.value = DESKTOP;
opt.textContent = `This computer (${desktopLabel})`;
portSelect.appendChild(opt);
};
+ const modeValue = () => desktopMode ? DESKTOP : teensyBootMode ? TEENSY_BOOT : null;
if (!pickedPort) {
// A neutral placeholder sits first and stays selected until the user chooses. It is
// what makes every pick a real `change` event, including re-picking the same option
@@ -716,11 +799,11 @@ document.addEventListener('DOMContentLoaded', () => {
if ("serial" in navigator) {
const pickOpt = document.createElement("option");
pickOpt.value = PICK_NEW;
- pickOpt.textContent = "An ESP32 on a USB port…";
+ pickOpt.textContent = "A board on a USB port…";
portSelect.appendChild(pickOpt);
}
addDesktopOpt();
- portSelect.value = desktopMode ? DESKTOP : IDLE;
+ portSelect.value = modeValue() || IDLE;
return;
}
// Picked state: name the selected port by the chip Detect read from it
@@ -737,7 +820,7 @@ document.addEventListener('DOMContentLoaded', () => {
pickOpt.textContent = "Pick another USB port…";
portSelect.appendChild(pickOpt);
addDesktopOpt();
- portSelect.value = desktopMode ? DESKTOP : "current";
+ portSelect.value = modeValue() || "current";
}
// Reflect the port state in the picker's Install gate after every rebuild
@@ -755,6 +838,7 @@ document.addEventListener('DOMContentLoaded', () => {
if (!ports.length || pickedPort) return;
pickedPort = ports[0];
desktopMode = false;
+ teensyBootMode = false;
syncPortState();
} catch {
// No grant, no port, or no permission: the picker stays as it was.
@@ -775,8 +859,9 @@ document.addEventListener('DOMContentLoaded', () => {
const eraseRow = document.getElementById("erase-row");
// The button's own control-row, so hiding it doesn't leave an empty row behind.
const monitorRow = document.getElementById("monitor-btn")?.closest(".control-row");
+ // A Teensy in its bootloader keeps the erase choice: a board whose settings went bad is the one that lands there.
if (eraseRow) eraseRow.hidden = desktopMode;
- if (monitorRow) monitorRow.hidden = desktopMode;
+ if (monitorRow) monitorRow.hidden = desktopMode || teensyBootMode;
}
syncPortState();
@@ -802,6 +887,7 @@ document.addEventListener('DOMContentLoaded', () => {
portPromptOpen = false;
}
syncPortState();
+ if (!granted) offerTeensyBootloader();
// Auto-detect right after a fresh grant — the ESP Web Tools / ESPHome
// model where picking the device detects it immediately, so the device
// list narrows without a second click. Only on a genuine new grant (not
@@ -817,12 +903,32 @@ document.addEventListener('DOMContentLoaded', () => {
}
}
+ // A Teensy with no serial port (a crashed program, another USB type) is not in the port list: after a canceled pick, the status line offers its bootloader instead, which WebHID reaches once the program button is pressed.
+ // Every canceled pick offers it, an earlier port included, since a cancel means the board the user looks for is missing from the list.
+ function offerTeensyBootloader() {
+ if (!("hid" in navigator) || teensyBootMode) return;
+ const status = document.getElementById("detect-status");
+ const link = document.createElement("a");
+ link.href = "#";
+ link.textContent = "Teensy not listed? Press its program button, then click here.";
+ link.addEventListener("click", (e) => {
+ e.preventDefault();
+ teensyBootMode = true;
+ desktopMode = false;
+ syncPortState();
+ // Narrow the device list to the Teensy, as a detected port would.
+ installPicker.runDetect((text) => { status.textContent = text; });
+ });
+ status.replaceChildren(link);
+ }
+
function applyPortChoice(value) {
- if (value === IDLE) return; // the placeholder is not a choice
- if (value === DESKTOP) { desktopMode = true; syncPortState(); return; }
+ if (value === IDLE || value === TEENSY_BOOT) return; // the placeholder, and the active bootloader mode, are not choices
+ if (value === DESKTOP) { desktopMode = true; teensyBootMode = false; syncPortState(); return; }
// Any other choice is about a device again. openPortPicker ends in its own
// syncPortState, so only sync here when nothing else will.
desktopMode = false;
+ teensyBootMode = false;
if (value === PICK_NEW) openPortPicker();
else syncPortState();
}
@@ -986,8 +1092,9 @@ document.addEventListener('DOMContentLoaded', () => {
// below re-evaluates the button whenever pickedPort changes.)
// Desktop mode has nothing to open, so the port gate does not apply: the button is a
// download and is enabled as soon as an archive for this OS exists.
- hasPort: () => desktopMode || !!pickedPort,
+ hasPort: () => desktopMode || teensyBootMode || !!pickedPort,
onDetect: async () => {
+ if (teensyBootMode) return TEENSY_FAMILY;
// The monitor (if open) holds the port — release it before esptool
// claims it, same as the install path. detect() keeps the port open
// afterwards so the following Install reuses it.
@@ -1034,8 +1141,8 @@ document.addEventListener('DOMContentLoaded', () => {
// (the build has no WIFI_SETTINGS RPC) and tell the user to connect Ethernet instead.
const ethOnly = /-eth$/.test(firmware);
await installer.start({
- // pickedPort may be null — orchestrator falls back to requestPort.
- port: pickedPort,
+ // pickedPort may be null, and the orchestrator falls back to requestPort. In Teensy bootloader mode there is no port.
+ port: teensyBootMode ? null : pickedPort,
manifestUrl: localUrl,
device, // names the install title + identifies the catalog entry
applyDefaults, // gates the APPLY_OP config push (not txPower, sent earlier)
@@ -1047,6 +1154,8 @@ document.addEventListener('DOMContentLoaded', () => {
uiWaitForIp,
uiShowNeedsIpRetrying: showNeedsIpRetrying,
uiWaitForPortRetry,
+ uiGrantBootloader,
+ uiFindBoard,
onSuccess: handleSuccess,
onError: handleError,
onLog: appendLog,
diff --git a/mooninstaller/teensy-writer.js b/mooninstaller/teensy-writer.js
new file mode 100644
index 000000000..7f98340f1
--- /dev/null
+++ b/mooninstaller/teensy-writer.js
@@ -0,0 +1,173 @@
+// The installer's Teensy 4.1 writer: PJRC's HalfKay bootloader protocol over WebHID, as teensy_loader_cli speaks it.
+//
+// Dependency-free and import-safe in node (browser APIs are touched only inside the async functions), so test/js pins the report layout.
+//
+// The protocol, from teensy_loader_cli.c:
+// - a running board enters its bootloader when its USB serial port is opened at 134 baud;
+// - the bootloader is a HID device, 16C0:0478, taking 1088-byte output reports: a 64-byte header whose first 3 bytes are the block's offset from the flash base, little-endian, then 1024 image bytes;
+// - block 0 goes first because it erases the chip, so the first writes wait up to 45 s and the rest 0.5 s;
+// - a block of only 0xFF after the first is skipped, since the chip is erased;
+// - a report whose first 3 bytes are 0xFF boots the new program.
+
+export const PJRC_VID = 0x16C0;
+export const TEENSY_SERIAL_PID = 0x0483; // a Teensy running a program built with USB type Serial, as MoonLight and the factory blink are
+export const HALFKAY_PID = 0x0478;
+export const TEENSY_FAMILY = "Teensy 4.1";
+
+const BLOCK_BYTES = 1024;
+const HEADER_BYTES = 64;
+export const REPORT_BYTES = HEADER_BYTES + BLOCK_BYTES;
+// The program flash HalfKay writes on a Teensy 4.1: 8 MB less the 256 KB the core reserves at the top.
+export const TEENSY41_CODE_BYTES = 8126464;
+// MoonLight's settings: a 1 MB LittleFS directly below that reserved 256 KB (platform_teensy.cpp, ProgramFlashFs), in 4 KB blocks.
+// Zeroing its first two blocks, the superblock pair, leaves nothing to mount, so the firmware formats it at boot: an erase of
+// every setting that leaves the program alone, which the bootloader cannot do otherwise, since it erases the program area only.
+export const TEENSY41_SETTINGS_OFFSET = TEENSY41_CODE_BYTES - 1024 * 1024;
+const SETTINGS_SUPERBLOCK_BYTES = 2 * 4096;
+const ERASE_WRITES = 5; // teensy_loader_cli's `block_count <= 4`: the writes the chip erase can stall
+const ERASE_TIMEOUT_MS = 45000;
+const BLOCK_TIMEOUT_MS = 500;
+const BOOTLOADER_BAUD = 134;
+
+export function isTeensySerial(info) {
+ return !!info && info.usbVendorId === PJRC_VID && info.usbProductId === TEENSY_SERIAL_PID;
+}
+
+// Why an image cannot go to a Teensy 4.1, or "" when it can: the flat image starts with the FlexSPI configuration block, "FCFB".
+export function imageProblem(image) {
+ if (!(image instanceof Uint8Array) || image.length === 0) return "the image is empty";
+ if (image[0] === 0x3A) return "that is a HEX file, the installer writes the .bin";
+ if (image.length < 4 || String.fromCharCode(...image.subarray(0, 4)) !== "FCFB") return "not a Teensy 4 image";
+ if (image.length > TEENSY41_CODE_BYTES) return `the image is ${image.length} bytes, more than the ${TEENSY41_CODE_BYTES} a Teensy 4.1 holds`;
+ return "";
+}
+
+// The reports that write an image, in order: block 0 always, every later block unless it is all 0xFF.
+export function halfkayReports(image) {
+ const reports = [];
+ for (let offset = 0; offset < image.length; offset += BLOCK_BYTES) {
+ const block = image.subarray(offset, offset + BLOCK_BYTES);
+ if (offset > 0 && block.every(b => b === 0xFF)) continue;
+ const report = new Uint8Array(REPORT_BYTES).fill(0);
+ report[0] = offset & 0xFF;
+ report[1] = (offset >> 8) & 0xFF;
+ report[2] = (offset >> 16) & 0xFF;
+ report.fill(0xFF, HEADER_BYTES); // a short last block reads as erased flash beyond the image
+ report.set(block, HEADER_BYTES);
+ reports.push({ offset, report });
+ }
+ return reports;
+}
+
+// The reports that zero the settings' superblock pair.
+export function eraseSettingsReports() {
+ const reports = [];
+ for (let offset = TEENSY41_SETTINGS_OFFSET; offset < TEENSY41_SETTINGS_OFFSET + SETTINGS_SUPERBLOCK_BYTES; offset += BLOCK_BYTES) {
+ const report = new Uint8Array(REPORT_BYTES).fill(0);
+ report[0] = offset & 0xFF;
+ report[1] = (offset >> 8) & 0xFF;
+ report[2] = (offset >> 16) & 0xFF;
+ reports.push({ offset, report });
+ }
+ return reports;
+}
+
+export function bootReport() {
+ const report = new Uint8Array(REPORT_BYTES).fill(0);
+ report[0] = report[1] = report[2] = 0xFF;
+ return report;
+}
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+// Asks a running Teensy into its bootloader: PJRC's core reboots when its serial port is set to 134 baud.
+export async function rebootToBootloader(port) {
+ try {
+ await port.open({ baudRate: BOOTLOADER_BAUD });
+ } catch (e) {
+ // Held open by the installer at another rate: close it and open it again at the bootloader's.
+ if (e && e.name === "InvalidStateError") {
+ try {
+ await port.close();
+ await port.open({ baudRate: BOOTLOADER_BAUD });
+ } catch (_) { /* held by a reader, or already rebooting: the bootloader step that follows says what to do */ }
+ }
+ }
+ await sleep(100);
+ try { await port.close(); } catch (_) { /* the device left the bus */ }
+}
+
+const HALFKAY_FILTER = { vendorId: PJRC_VID, productId: HALFKAY_PID };
+
+// The browser's device chooser for the bootloader; it needs a user gesture, so call it inside a click. Null when the user cancels.
+// Chrome does not remember the grant (seen on the bench, although the bootloader reports a USB serial number): every install asks.
+export async function requestHalfKay() {
+ const devices = await navigator.hid.requestDevice({ filters: [HALFKAY_FILTER] });
+ return devices && devices.length ? devices[0] : null;
+}
+
+// One report, retried while the bootloader is busy (erasing), as teensy_loader_cli's teensy_write retries until its timeout.
+async function sendReport(device, report, timeoutMs) {
+ const deadline = Date.now() + timeoutMs;
+ for (;;) {
+ const remaining = deadline - Date.now();
+ try {
+ await Promise.race([
+ device.sendReport(0, report),
+ sleep(Math.max(remaining, 1)).then(() => { throw new Error("timeout"); }),
+ ]);
+ return;
+ } catch (e) {
+ if (Date.now() >= deadline) throw new Error(`the Teensy bootloader stopped answering (${e && e.message || e})`);
+ await sleep(10);
+ }
+ }
+}
+
+// Writes an image through an opened or unopened HalfKay device, with `eraseSettings` also the settings, and boots it; onProgress gets 0..100.
+export async function writeImage(device, image, { onProgress, onLog, eraseSettings = false } = {}) {
+ const problem = imageProblem(image);
+ if (problem) throw new Error(problem);
+ if (!device.opened) await device.open();
+ const reports = halfkayReports(image);
+ if (eraseSettings) {
+ if (onLog) onLog("[teensy] erasing the settings as well");
+ reports.push(...eraseSettingsReports());
+ }
+ if (onLog) onLog(`[teensy] writing ${reports.length} blocks of ${BLOCK_BYTES} bytes`);
+ for (let i = 0; i < reports.length; i++) {
+ await sendReport(device, reports[i].report, i < ERASE_WRITES ? ERASE_TIMEOUT_MS : BLOCK_TIMEOUT_MS);
+ if (onProgress) onProgress(Math.round(((i + 1) * 100) / reports.length));
+ }
+ if (onLog) onLog("[teensy] booting the new program");
+ try {
+ await device.sendReport(0, bootReport());
+ } catch (_) {
+ // The bootloader may leave the bus before it acknowledges the boot report.
+ }
+ try { await device.close(); } catch (_) { /* already gone */ }
+}
+
+// The browser's port chooser narrowed to a running Teensy; it needs a user gesture, so call it inside a click.
+export async function requestTeensySerial() {
+ return await navigator.serial.requestPort({ filters: [{ usbVendorId: PJRC_VID, usbProductId: TEENSY_SERIAL_PID }] });
+}
+
+// The granted Teensy serial ports of every board but the one being installed, noted before the install.
+export async function otherTeensyPorts(installing) {
+ const ports = await navigator.serial.getPorts();
+ return ports.filter(p => p !== installing && isTeensySerial(p.getInfo()));
+}
+
+// The installed board's serial port after the reboot, which re-enumerates; null when none appears in `waitMs`.
+// A port reports only its USB ids, which every Teensy shares, so the board is the Teensy port that is none of `others`: another granted board is never read or sent defaults.
+export async function rebootedSerialPort(waitMs, others = []) {
+ const deadline = Date.now() + waitMs;
+ for (;;) {
+ const ports = await navigator.serial.getPorts();
+ const port = ports.find(p => isTeensySerial(p.getInfo()) && !others.includes(p));
+ if (port) return port;
+ if (Date.now() >= deadline) return null;
+ await sleep(500);
+ }
+}
diff --git a/moonlive/effects/breathe.mle b/moonlive/effects/breathe.mle
index 7433af660..28b8ee764 100644
--- a/moonlive/effects/breathe.mle
+++ b/moonlive/effects/breathe.mle
@@ -20,6 +20,6 @@ class BreatheEffect {
void tick() {
int bri = floorBri + div(beatsin(bpm, t, 255) * (255 - floorBri), 255);
int p = hue + scale(beat(div(bpm, 2) + 1, t), drift + 1);
- fill(paletteR(p, bri), paletteG(p, bri), paletteB(p, bri));
+ fill(paletteRGB(p, bri));
}
}
diff --git a/moonlive/effects/chase.mle b/moonlive/effects/chase.mle
index d65213f83..04d2718f2 100644
--- a/moonlive/effects/chase.mle
+++ b/moonlive/effects/chase.mle
@@ -28,7 +28,7 @@ class ChaseEffect {
if (d < 0) { d = d + n; }
if (d < spread) {
int bri = 255 - div(d * tail, spread);
- setRGB(i, paletteR(hue + d, bri), paletteG(hue + d, bri), paletteB(hue + d, bri));
+ setRGB(i, paletteRGB(hue + d, bri));
}
}
}
diff --git a/moonlive/effects/ember.mle b/moonlive/effects/ember.mle
index a589e34d6..da1006a88 100644
--- a/moonlive/effects/ember.mle
+++ b/moonlive/effects/ember.mle
@@ -18,7 +18,7 @@ class EmberEffect {
void tick() {
for (int i = 0; i < 16; i = i + 1) {
- if (heat[i] > cool) { heat[i] = heat[i] - cool; } else { heat[i] = 0; }
+ heat[i] = max(heat[i] - cool, 0);
}
for (int j = 0; j < 16; j = j + 1) {
@@ -26,8 +26,7 @@ class EmberEffect {
}
for (int k = 0; k < 16; k = k + 1) {
- setRGB(k, paletteR(heat[k], heat[k]), paletteG(heat[k], heat[k]),
- paletteB(heat[k], heat[k]));
+ setRGB(k, paletteRGB(heat[k], heat[k]));
}
}
}
diff --git a/moonlive/effects/glow.mle b/moonlive/effects/glow.mle
index ebeb8a9e4..aa8524853 100644
--- a/moonlive/effects/glow.mle
+++ b/moonlive/effects/glow.mle
@@ -25,18 +25,18 @@ class GlowEffect {
void tick() {
int now = div((audioBand(0) + audioBand(1) + audioBand(2) + audioBand(3)) * gain, 400);
- if (now > 255) { now = 255; }
+ now = min(now, 255);
if (now * 16 > bass) { bass = bass + div(now * 16 - bass, 2); }
else { bass = bass - div((bass - now * 16) * decay + 99, 100); }
avg = avg + div(now * 16 - avg, 32);
int target = floorBri + div(div(bass, 16) * (255 - floorBri), 255);
int kick = 0;
- if (now > 40) { if (now * 16 > avg + div(avg, 2)) { kick = 1; } }
- if (kick > kicking) { bri = bri + flare; }
+ if (now > 40 && now * 16 > avg + div(avg, 2)) { kick = 1; }
+ if (kick && !kicking) { bri = bri + flare; }
kicking = kick;
if (bri > target) { bri = bri - div((bri - target) * decay + 99, 100); } else { bri = target; }
- if (bri > 255) { bri = 255; }
+ bri = min(bri, 255);
int low = 0;
int most = 0;
@@ -47,6 +47,6 @@ class GlowEffect {
hue = hue + div((low * 24 + div(t, 400)) * 16 - hue, 64);
int p = mod(div(hue, 16), 256);
- fill(paletteR(p, bri), paletteG(p, bri), paletteB(p, bri));
+ fill(paletteRGB(p, bri));
}
}
diff --git a/moonlive/effects/pulse.mle b/moonlive/effects/pulse.mle
index f56ca022c..0665013cf 100644
--- a/moonlive/effects/pulse.mle
+++ b/moonlive/effects/pulse.mle
@@ -19,14 +19,11 @@ class PulseEffect {
}
void tick() {
- if (audioBeat() > 0) {
+ if (audioBeat()) {
lit = 255;
hue = hue + hueStep;
}
- if (lit > decay) { lit = lit - decay; }
- else { lit = 0; }
-
- if (lit < floorBri) { lit = floorBri; }
- fill(paletteR(hue, lit), paletteG(hue, lit), paletteB(hue, lit));
+ lit = max(lit - decay, floorBri);
+ fill(paletteRGB(hue, lit));
}
}
diff --git a/moonlive/effects/sparkle.mle b/moonlive/effects/sparkle.mle
index e43cdc383..3180588d1 100644
--- a/moonlive/effects/sparkle.mle
+++ b/moonlive/effects/sparkle.mle
@@ -19,9 +19,8 @@ class SparkleEffect {
fade(fadeAmt);
int n = width * height * depth;
for (int i = 0; i < density; i = i + 1) {
- int p = random16(256);
- if (p > hueSpread) { p = hueSpread; }
- setRGB(random16(n), paletteR(p, 255), paletteG(p, 255), paletteB(p, 255));
+ int p = min(random16(256), hueSpread);
+ setRGB(random16(n), paletteRGB(p, 255));
}
}
}
diff --git a/moonlive/effects/spectrum.mle b/moonlive/effects/spectrum.mle
index b3192b291..106bd9376 100644
--- a/moonlive/effects/spectrum.mle
+++ b/moonlive/effects/spectrum.mle
@@ -22,21 +22,15 @@ class SpectrumEffect {
if (width < 2) { bars = height; }
for (int x = 0; x < bars; x = x + 1) {
int b = div(x * 16, bars);
- int mag = div(audioBand(b) * gain, 100);
- if (mag > 255) { mag = 255; }
- if (width < 2) {
- setRGB(x, div(paletteR(b * 16, 255) * mag, 255), div(paletteG(b * 16, 255) * mag, 255),
- div(paletteB(b * 16, 255) * mag, 255));
- }
+ int mag = min(div(audioBand(b) * gain, 100), 255);
+ RGB c = paletteRGB(b * 16, 255);
+ if (width < 2) { setRGB(x, div(c.r * mag, 255), div(c.g * mag, 255), div(c.b * mag, 255)); }
int top = div(mag * height, 256);
if (width < 2) { top = 0; }
for (int y = 0; y < top; y = y + 1) {
- setRGB((height - 1 - y) * width + x, paletteR(b * 16, 255), paletteG(b * 16, 255),
- paletteB(b * 16, 255));
- }
- if (peakHold > 0) {
- if (top > 0) { setRGB((height - top) * width + x, 255, 255, 255); }
+ setRGB((height - 1 - y) * width + x, c);
}
+ if (peakHold && top > 0) { setRGB((height - top) * width + x, 255, 255, 255); }
}
}
}
diff --git a/src/core/module/Control.cpp b/src/core/module/Control.cpp
index 67528b3d8..939516538 100644
--- a/src/core/module/Control.cpp
+++ b/src/core/module/Control.cpp
@@ -218,9 +218,9 @@ void writeControlMetadata(JsonSink& sink, const ControlDescriptor& c) {
return;
}
case ControlType::Progress:
- // `bytes` (in min, see addProgress): 1 → KB label, 0 → plain count.
- sink.appendf(",\"total\":%lu,\"bytes\":%s", static_cast(c.aux),
- c.min ? "true" : "false");
+ // The unit rides in min (see addProgress); the UI scales by it and picks KB, MB or GB itself.
+ sink.appendf(",\"total\":%lu,\"unit\":\"%s\"", static_cast(c.aux),
+ static_cast(c.min) == ProgressUnit::KiB ? "KiB" : "B");
return;
case ControlType::List: {
// The detail an expanded row shows, as an array parallel to the summary `value`, which stays small for the collapsed list.
diff --git a/src/core/module/Control.h b/src/core/module/Control.h
index c5d1e1107..d7c3c1bc7 100644
--- a/src/core/module/Control.h
+++ b/src/core/module/Control.h
@@ -67,6 +67,12 @@ enum class ControlType : uint8_t {
Palette ///< a palette dropdown, whose options carry their own swatch colors
};
+/// The unit a progress control's value and total count in; the UI sizes the label from it.
+enum class ProgressUnit : uint8_t {
+ Bytes, ///< a byte count, which fits a uint32_t up to 4 GiB
+ KiB ///< kibibytes, for a store whose byte count overruns a uint32_t
+};
+
class JsonSink; // forward-declared so the descriptor can hold a pointer
/// Emits a palette control's options, which the light domain owns and core calls.
@@ -321,10 +327,10 @@ class ControlList {
controls_[count_++] = {&var, name, reinterpret_cast(options), ControlType::Select, 0, optionCount};
}
- /// Bind a value as a progress bar against a total, labeled either as bytes or as a count.
- void addProgress(const char* name, uint32_t& var, uint32_t total, bool bytes = true) {
+ /// Bind a value as a progress bar against a total, both counted in `unit`.
+ void addProgress(const char* name, uint32_t& var, uint32_t total, ProgressUnit unit = ProgressUnit::Bytes) {
grow();
- controls_[count_++] = {&var, name, total, ControlType::Progress, bytes ? 1 : 0, 0};
+ controls_[count_++] = {&var, name, total, ControlType::Progress, static_cast(unit), 0};
}
/// Bind four octets as an address, which serializes as its dotted-quad string.
diff --git a/src/core/moonlive/MoonLive.cpp b/src/core/moonlive/MoonLive.cpp
index 6cd3c74da..9943875e2 100644
--- a/src/core/moonlive/MoonLive.cpp
+++ b/src/core/moonlive/MoonLive.cpp
@@ -74,7 +74,7 @@ bool MoonLive::compile(uint8_t r, uint8_t g, uint8_t b) {
size_t len = emitFill(staging.p, staging.n, r, g, b);
void* block = place(staging.p, len);
if (!block) return false;
- fn_ = reinterpret_cast(block);
+ fn_ = asFunction(block);
return true;
}
@@ -118,7 +118,7 @@ bool MoonLive::compile(const char* source, const BuiltinTable& table, const SysV
entries_[i] = {entryNames_[i], n, cr.entries[i].offset, cr.entries[i].ret};
}
stringLen_ = cr.stringLen;
- ctrl_ = reinterpret_cast(block);
+ ctrl_ = asFunction(block);
return true;
}
@@ -183,7 +183,7 @@ bool MoonLive::compileAnimated() {
size_t len = emitAnimatedFill(staging.p, staging.n);
void* block = place(staging.p, len);
if (!block) return false;
- anim_ = reinterpret_cast(block);
+ anim_ = asFunction(block);
return true;
}
diff --git a/src/core/moonlive/MoonLive.h b/src/core/moonlive/MoonLive.h
index d07cc21ee..593a5af9e 100644
--- a/src/core/moonlive/MoonLive.h
+++ b/src/core/moonlive/MoonLive.h
@@ -42,6 +42,12 @@ class MoonLive {
/// Compile the animated routine, whose color derives from the per-frame `t`.
bool compileAnimated();
+ /// Emitted code at `code` as a function of type Fn, its address carrying what the target's call needs.
+ template
+ static Fn asFunction(const void* code) {
+ return code ? reinterpret_cast(reinterpret_cast(code) | platform::codeAddressBits) : nullptr;
+ }
+
/// Whether a program is compiled and callable.
bool ok() const { return fn_ != nullptr || anim_ != nullptr || ctrl_ != nullptr; }
@@ -49,7 +55,7 @@ class MoonLive {
/// The compiled function named `name`, or null when the script defined none.
CtrlFn entry(const char* name) const {
const uint8_t* p = entryCode(name);
- return p ? reinterpret_cast(reinterpret_cast(p)) : nullptr;
+ return asFunction(p);
}
/// What a named entry point declared it returns, or Void when the script has no such function.
@@ -114,7 +120,7 @@ class MoonLive {
// Through the code address, since casting between return types is what the warning catches.
const uint8_t* code = entryCode(name);
if (!code) return fallback;
- ValueFn f = reinterpret_cast(reinterpret_cast(code));
+ ValueFn f = asFunction(code);
ctrlArena_[kDepthSlot] = 0; // same fresh-depth contract as run()
return f(buf, nLights, cpl, t, ctrlArena_);
}
diff --git a/src/core/moonlive/MoonLiveBuiltins.h b/src/core/moonlive/MoonLiveBuiltins.h
index 3f759b5d9..d737cb581 100644
--- a/src/core/moonlive/MoonLiveBuiltins.h
+++ b/src/core/moonlive/MoonLiveBuiltins.h
@@ -69,6 +69,26 @@ struct Builtin {
uint8_t fixedArgs = 0;
/// Whether the result is fixed-point.
bool fixedReturn = false;
+ /// The struct this builtin answers with, by name, written into its first argument slots; null for a scalar or no result.
+ const char* retStruct = nullptr;
+ /// Whether the arguments may be all whole or all fixed, with the result following them, as `min` and `abs` are.
+ bool sameTypeArgs = false;
+};
+
+/// Most fields a struct type carries: three for a color or a coordinate, four for a color with white.
+static constexpr uint8_t kMaxStructFields = 4;
+
+// Host-registered like the builtins, so the core compiler names no domain's types.
+/// A struct type a script can declare: a fixed number of fields of one scalar type.
+struct StructDef {
+ /// The type name a script writes, such as `RGB`.
+ const char* name = nullptr;
+ /// The scalar type every field has.
+ CtrlType type = CtrlType::Int;
+ /// How many fields it has.
+ uint8_t fieldCount = 0;
+ /// The field names, in storage order.
+ const char* fields[kMaxStructFields] = {};
};
// An overflow otherwise surfaces as "unknown function" for a builtin that plainly exists.
@@ -93,6 +113,32 @@ struct BuiltinTable {
uint8_t count = 0;
/// Set when an `add` was dropped for lack of room.
bool overflowed = false;
+ /// How many struct types one table defines.
+ static constexpr uint8_t kMaxStructs = 4;
+ /// The struct types a script may declare.
+ StructDef structs[kMaxStructs];
+ /// How many struct types are registered.
+ uint8_t structCount = 0;
+
+ /// Register a struct type, returning false when the table is full or the type is malformed.
+ bool addStruct(const StructDef& s) {
+ if (!s.name || s.fieldCount == 0 || s.fieldCount > kMaxStructFields) return false;
+ for (uint8_t f = 0; f < s.fieldCount; f++) if (!s.fields[f]) return false; // a field lookup reads every name
+ if (structCount >= kMaxStructs) { overflowed = true; return false; }
+ structs[structCount++] = s;
+ return true;
+ }
+
+ /// The struct type registered under `name`, or null when there is none.
+ const StructDef* findStruct(const char* name, size_t len) const {
+ for (uint8_t i = 0; i < structCount; i++) {
+ const char* n = structs[i].name;
+ size_t j = 0;
+ for (; j < len && n[j]; j++) if (n[j] != name[j]) break;
+ if (j == len && n[j] == 0) return &structs[i];
+ }
+ return nullptr;
+ }
/// Register one builtin, returning false when the table is full.
bool add(const Builtin& b) {
@@ -150,7 +196,7 @@ static constexpr uint8_t kMaxCallDepth = 32;
// The arena rather than the frame, since a callee copies its arguments out as its first act.
/// Where a script call's arguments are passed.
-static constexpr uint8_t kMaxScriptArgs = 4; // per call; a helper wanting more wants an array
+static constexpr uint8_t kMaxScriptArgs = 8; // words per call, a struct taking one per field
static constexpr uint8_t kScriptArgBytes = 4; // one machine word, as a frame slot is
/// Where the argument block starts, aligned to four bytes because these are 32-bit slots.
static constexpr uint8_t kScriptArgBase = static_cast((kDepthSlot + 1 + 3) & ~3);
diff --git a/src/core/moonlive/MoonLiveBuiltins_common.h b/src/core/moonlive/MoonLiveBuiltins_common.h
index 06c8c03b0..64861966e 100644
--- a/src/core/moonlive/MoonLiveBuiltins_common.h
+++ b/src/core/moonlive/MoonLiveBuiltins_common.h
@@ -10,6 +10,7 @@
#include
#include
#include
+#include // the same-type builtins register from a braced list
/// @defgroup moonlive_builtins_common MoonLive neutral builtins
/// @{
@@ -203,6 +204,28 @@ extern "C" inline uint32_t mm_ml_addSelect(const uintptr_t* args, uint32_t, cons
return 0;
}
+// Signed, and each works on fixed values too, since Q16.16 keeps the order and the sign of a whole number.
+/// The magnitude of a value, widened so the most negative int does not overflow.
+extern "C" inline uint32_t mm_ml_abs(const uintptr_t* args, uint32_t, const uint8_t*) {
+ const int64_t v = signedArg(args[0]);
+ return static_cast(v < 0 ? -v : v);
+}
+/// The smaller of two values.
+extern "C" inline uint32_t mm_ml_min(const uintptr_t* args, uint32_t, const uint8_t*) {
+ const int32_t a = signedArg(args[0]), b = signedArg(args[1]);
+ return static_cast(a < b ? a : b);
+}
+/// The larger of two values.
+extern "C" inline uint32_t mm_ml_max(const uintptr_t* args, uint32_t, const uint8_t*) {
+ const int32_t a = signedArg(args[0]), b = signedArg(args[1]);
+ return static_cast(a > b ? a : b);
+}
+/// A value held between a low and a high bound.
+extern "C" inline uint32_t mm_ml_clamp(const uintptr_t* args, uint32_t, const uint8_t*) {
+ const int32_t v = signedArg(args[0]), lo = signedArg(args[1]), hi = signedArg(args[2]);
+ return static_cast(v < lo ? lo : (v > hi ? hi : v));
+}
+
// Both vocabularies call this first and then add their own, so a name means one thing everywhere.
/// Register the neutral builtins into whatever table asks.
inline void addCommonBuiltins(BuiltinTable& t) {
@@ -230,14 +253,27 @@ inline void addCommonBuiltins(BuiltinTable& t) {
// addSelect declares a dropdown: a name, the member, and its option names in one quoted "a|b|c".
t.add({"addSelect", 3, /*returns*/ false, BuiltinKind::Call, &mm_ml_addSelect, {},
/*byRef*/ 0x2, /*byStr*/ 0x5});
+ // The arithmetic a shape needs, whole or fixed: the arguments agree and the result follows them.
+ for (const Builtin& b : {Builtin{"abs", 1, true, BuiltinKind::Call, &mm_ml_abs, {}},
+ Builtin{"min", 2, true, BuiltinKind::Call, &mm_ml_min, {}},
+ Builtin{"max", 2, true, BuiltinKind::Call, &mm_ml_max, {}},
+ Builtin{"clamp", 3, true, BuiltinKind::Call, &mm_ml_clamp, {}}}) {
+ Builtin same = b;
+ same.sameTypeArgs = true;
+ t.add(same);
+ }
}
// Run once after a successful compile, where a compiled module's defineControls() sits.
/// The entry a binding runs so the controls a script declares exist.
inline constexpr const char* kEntryDefineControls = "defineControls";
-/// What a binding attaches around a defineControls run, for the sizing its own builtins ask for.
-using DefineHook = void (*)(void* ctx, bool attach);
+// Before the compile, so the last program's buffers are gone before the next one needs memory.
+/// The moments a binding is told about: a compile starting, and the defineControls run that sizes the new program's buffers.
+enum class DefinePhase : uint8_t { Compiling, Attach, Detach };
+
+/// What a binding does at each of those moments, for the sizing its own builtins ask for.
+using DefineHook = void (*)(void* ctx, DefinePhase phase);
// Re-runnable like its compiled counterpart, since the declared list is cleared first.
/// Run a script's defineControls, so the controls it declares exist.
@@ -248,12 +284,12 @@ inline void runDefineControls(MoonLive& engine, DefineHook hook = nullptr, void*
int32_t lo, int32_t hi, CtrlType type, const char* options) {
static_cast(ctx)->addDeclaredControl(n, off, lo, hi, type, options);
}, &engine)) return;
- if (hook) hook(hookCtx, true);
+ if (hook) hook(hookCtx, DefinePhase::Attach);
engine.clearDeclaredControls(); // re-runnable: rebuild rather than append
// This entry point writes no pixels, but `run` refuses a null or undersized buffer.
uint8_t scratch[3] = {};
engine.run(scratch, 1, 3, 0, kEntryDefineControls);
- if (hook) hook(hookCtx, false);
+ if (hook) hook(hookCtx, DefinePhase::Detach);
setAddControlSink(nullptr, nullptr);
}
diff --git a/src/core/moonlive/MoonLiveCompiler.cpp b/src/core/moonlive/MoonLiveCompiler.cpp
index 6a508a9ae..bf8725dea 100644
--- a/src/core/moonlive/MoonLiveCompiler.cpp
+++ b/src/core/moonlive/MoonLiveCompiler.cpp
@@ -17,7 +17,7 @@ namespace {
/// Every token the grammar has.
enum class Tok { Ident, Number, String, Assign, LParen, RParen, LBrace, RBrace, Comma, Semicolon,
Plus, Minus, Star, Slash, Percent, Less, LessEq, Greater, GreaterEq, EqEq, NotEq,
- LBracket, RBracket, End, Error };
+ LBracket, RBracket, Dot, AndAnd, OrOr, Not, End, Error };
/// One token at a time over the source, with no buffer of its own.
struct Lexer {
@@ -96,6 +96,11 @@ struct Lexer {
if (c == '>' && p[1] == '=') { p += 2; kind = Tok::GreaterEq; return; }
if (c == '=' && p[1] == '=') { p += 2; kind = Tok::EqEq; return; }
if (c == '!' && p[1] == '=') { p += 2; kind = Tok::NotEq; return; }
+ if (c == '&' && p[1] == '&') { p += 2; kind = Tok::AndAnd; return; }
+ if (c == '|' && p[1] == '|') { p += 2; kind = Tok::OrOr; return; }
+ if (c == '!') { p++; kind = Tok::Not; return; }
+ // A field access; a number's own point is read with the number, which starts with a digit.
+ if (c == '.') { p++; kind = Tok::Dot; return; }
if (c == '=') { p++; kind = Tok::Assign; return; }
if (c == '(') { p++; kind = Tok::LParen; return; }
if (c == ')') { p++; kind = Tok::RParen; return; }
@@ -142,6 +147,13 @@ struct Lexer {
// --- Parser to IR --------------------------------------------------------------------
// It knows the grammar and resolves call names against the injected table, owning no function name.
+/// What a condition operator met where a value is wanted says.
+constexpr const char* kJoinsConditions = "&&, || and ! join the conditions of an if, not values";
+/// What a whole struct where one value belongs says.
+constexpr const char* kOneFieldAtATime = "a struct is read one field at a time: write name.field";
+/// What a comparison after a negated value says.
+constexpr const char* kNegatesWhatFollows = "! negates what follows it: write !(a < b) to negate a comparison";
+
/// Recursive descent, evaluating each expression into a virtual register and emitting IR.
struct Parser {
Lexer& lex; ///< the token stream being parsed
@@ -155,11 +167,18 @@ struct Parser {
uint8_t nameLen; ///< how long that span is
uint16_t irStart; ///< the IR index its body starts at
RetType ret; ///< what it declares it returns
- uint8_t params; }; ///< how many arguments it declares
+ uint8_t params; ///< how many argument words it declares
+ uint8_t retStruct; ///< the struct it returns, as its table index plus one, or 0
+ uint8_t argCount; ///< how many parameters it declares, a struct counting once
+ uint8_t argStruct[moonlive::kMaxScriptArgs]; }; ///< each parameter's struct, its table index plus one, or 0 for a scalar
FnMark fns[kMaxEntryPoints] = {}; ///< the functions parsed so far
uint8_t fnCount = 0; ///< how many of them there are
/// What the function being parsed returns, so a `return` is checked where it is written.
RetType curRet = RetType::Void;
+ /// The struct it returns, when it returns one.
+ const StructDef* curRetStruct = nullptr;
+ /// Each argument word's type, so the entry copy narrows a byte field as it narrows a byte parameter.
+ CtrlType paramWordType[kMaxScriptArgs] = {};
VReg nextTemp = kFirstTemp; ///< the high-water mark, also vregsUsed
VReg freeStack[kMaxVRegs] = {}; ///< recycled temps, so a dead vreg is reused
uint8_t freeCount = 0; ///< how many are on that stack
@@ -167,8 +186,9 @@ struct Parser {
/// A script-local variable, such as a `for` counter, which the script writes.
struct Local { const char* name; ///< its name, a span into the source
size_t nameLen; ///< how long that span is
- uint8_t slot; ///< the frame slot holding it
- CtrlType type; }; ///< its declared type, which carries the scaling
+ uint8_t slot; ///< the frame slot holding it, the first field's for a struct
+ CtrlType type; ///< its declared type, a struct's field type
+ uint8_t st = 0; }; ///< its struct type, as its table index plus one, or 0 for a scalar
Local locals[kMaxLocals] = {}; ///< the locals currently in scope
uint8_t localCount = 0; ///< how many of them there are
uint8_t slotHighWater = 0; ///< slots currently in scope
@@ -180,9 +200,12 @@ struct Parser {
uint16_t stringCap = 0; ///< its capacity
uint16_t stringLen = 0; ///< how much of it is used
DeclaredControl members[kMaxCtrls] = {}; ///< the members declared so far
+ uint8_t memberStruct[kMaxCtrls] = {}; ///< a member's struct type, as its table index plus one, or 0
// No AST, so the type rides alongside the register and every parse function sets it.
/// Whether the value last parsed is Q16.16 rather than a plain integer.
bool exprIsFixed = false;
+ /// Whether a same-type builtin's arguments, last parsed, were fixed, which its result follows.
+ bool argsFixed = false;
// A literal converts by patching its Const, since its meaning is visible at the call site.
/// The Const op index when the value last parsed is one integer literal, else -1.
int exprLitConst = -1;
@@ -244,6 +267,203 @@ struct Parser {
return -1;
}
+ // --- Structs: a host-registered type is its fields under one name, a named array ---
+
+ // A byte rather than a pointer in every local, member and function record, since the parser lives on the compiling task's stack.
+ /// The struct a stored index names, or null for 0.
+ const StructDef* structAt(uint8_t idx) const { return idx ? &table.structs[idx - 1] : nullptr; }
+ /// The index a struct is stored as: its place in the table plus one, or 0 for none.
+ uint8_t structIndex(const StructDef* st) const { return st ? static_cast(st - table.structs + 1) : 0; }
+
+ /// A field's index in a struct, or -1.
+ static int fieldIndex(const StructDef& st, const char* n, size_t len) {
+ for (uint8_t f = 0; f < st.fieldCount; f++)
+ if (std::strlen(st.fields[f]) == len && std::strncmp(st.fields[f], n, len) == 0) return f;
+ return -1;
+ }
+
+ /// Parse `.field` after a struct's name, answering the field's index, or -1 having failed.
+ int parseField(const StructDef& st) {
+ if (!expect(Tok::Dot, kOneFieldAtATime)) return -1;
+ if (lex.kind != Tok::Ident) { fail("expected a field name after '.'"); return -1; }
+ const int f = fieldIndex(st, lex.identBeg, lex.identLen);
+ if (f < 0) { fail("this struct has no field of that name"); return -1; }
+ lex.advance();
+ return f;
+ }
+
+ /// The arena byte of a struct member's field.
+ uint8_t memberFieldOffset(int mi, int f) const {
+ return static_cast(members[mi].offset + f * ctrlWidth(members[mi].type));
+ }
+
+ /// Load one field of a struct member, at its own width.
+ VReg loadMemberField(int mi, int f) {
+ VReg v = alloc();
+ emit({ctrlWidth(members[mi].type) == 1 ? IrOp::LoadCtrl : IrOp::LoadCtrl32, v, 0,0,0,0,
+ memberFieldOffset(mi, f), nullptr, {}});
+ return v;
+ }
+
+ // Peeks without consuming, so a caller can ask before it commits to a scalar or a struct.
+ /// The struct type the expression at the cursor has as a whole value, or null for a scalar.
+ const StructDef* structValueAt() const {
+ if (lex.kind != Tok::Ident) return nullptr;
+ Lexer probe = lex;
+ probe.advance();
+ // A call or constructor is followed past its parentheses, so `getRGB(i).r` reads as the scalar it is.
+ if (probe.kind == Tok::LParen) {
+ // A lexer error does not move past its character, so it ends the probe as the end does.
+ for (int depth = 0; probe.kind != Tok::End && probe.kind != Tok::Error; ) {
+ if (probe.kind == Tok::LParen) depth++;
+ if (probe.kind == Tok::RParen && --depth == 0) { probe.advance(); break; }
+ probe.advance();
+ }
+ }
+ if (probe.kind == Tok::Dot) return nullptr; // `c.r` is one field, a scalar
+ if (const StructDef* st = table.findStruct(lex.identBeg, lex.identLen)) return st; // a constructor
+ const int li = findLocal(lex.identBeg, lex.identLen);
+ if (li >= 0) return structAt(locals[li].st);
+ const int mi = findMember(lex.identBeg, lex.identLen);
+ if (mi >= 0) return structAt(memberStruct[mi]);
+ return structCallAt(); // a builtin or a script function answering one
+ }
+
+ /// Keep a struct answer's slots, the first `st.fieldCount` from `argBase`, which the builtin writes past its arguments; false when they do not fit.
+ bool reserveStructAnswer(uint8_t argBase, const StructDef& st) {
+ if (argBase + st.fieldCount > kMaxLocals) { fail("too many arguments to hold"); return false; }
+ if (slotHighWater < argBase + st.fieldCount) slotHighWater = static_cast(argBase + st.fieldCount);
+ if (slotHighWater > slotsUsed) slotsUsed = slotHighWater;
+ return true;
+ }
+
+ /// The struct a call or constructor at the cursor answers, read through a following `.field`.
+ const StructDef* structCallAt() const {
+ if (lex.kind != Tok::Ident) return nullptr;
+ if (const StructDef* st = table.findStruct(lex.identBeg, lex.identLen)) return st;
+ if (const Builtin* b = table.find(lex.identBeg, lex.identLen))
+ return b->retStruct ? table.findStruct(b->retStruct, std::strlen(b->retStruct)) : nullptr;
+ for (uint8_t i = 0; i < fnCount; i++)
+ if (fns[i].nameLen == lex.identLen && std::strncmp(fns[i].name, lex.identBeg, lex.identLen) == 0)
+ return structAt(fns[i].retStruct);
+ return nullptr;
+ }
+
+ // Every source of a struct value lands in registers, one per field, so every destination stores the same way.
+ /// Parse a whole struct value of type `st` into one register per field.
+ bool parseStructValue(const StructDef& st, VReg out[kMaxStructFields], bool thenField = false) {
+ const StructDef* got = thenField ? structCallAt() : structValueAt();
+ if (got != &st) {
+ fail(got ? "this is a different struct type"
+ : "expected a struct value: a constructor, a struct variable or a call answering one");
+ return false;
+ }
+ if (table.findStruct(lex.identBeg, lex.identLen)) { // `RGB(r, g, b)`: one expression per field
+ lex.advance();
+ if (!expect(Tok::LParen, "expected '(' after the struct type")) return false;
+ for (uint8_t f = 0; f < st.fieldCount; f++) {
+ if (f > 0 && !expect(Tok::Comma, "expected ',' between the fields")) return false;
+ out[f] = parseExpr();
+ if (failed) return false;
+ if (exprIsFixed) { fail("a struct field is a whole number: write toInt(x)"); return false; }
+ if (exprLitConst >= 0 && !fitsInType(ir.ops[exprLitConst].imm, st.type))
+ { fail("this number is out of range for that field"); return false; }
+ narrowToType(out[f], st.type);
+ }
+ if (lex.kind == Tok::Comma) { fail("too many fields for this struct"); return false; }
+ return expect(Tok::RParen, "expected ')' to close the struct");
+ }
+ const int li = findLocal(lex.identBeg, lex.identLen);
+ if (li >= 0) {
+ lex.advance();
+ for (uint8_t f = 0; f < st.fieldCount; f++) {
+ out[f] = alloc();
+ emit({IrOp::Reload, out[f], 0,0,0,0, locals[li].slot + f, nullptr, {}});
+ }
+ return true;
+ }
+ const int mi = findMember(lex.identBeg, lex.identLen);
+ if (mi >= 0) {
+ lex.advance();
+ for (uint8_t f = 0; f < st.fieldCount; f++) out[f] = loadMemberField(mi, f);
+ return true;
+ }
+ if (const Builtin* b = table.find(lex.identBeg, lex.identLen)) {
+ // The builtin writes its fields into its first argument slots, which are reserved even past its arguments.
+ lex.advance();
+ if (!expect(Tok::LParen, "expected '(' after the function name")) return false;
+ const uint8_t argBase = slotHighWater;
+ uint8_t n = 0;
+ if (!parseBuiltinArgs(b, n)) return false;
+ if (!reserveStructAnswer(argBase, st)) return false;
+ VReg r = alloc();
+ emit({IrOp::Call, r, 0, n, 0, 0, argBase, b->fn, {}});
+ freeTemp(r);
+ for (uint8_t f = 0; f < st.fieldCount; f++) {
+ out[f] = alloc();
+ emit({IrOp::Reload, out[f], 0,0,0,0, argBase + f, nullptr, {}});
+ }
+ slotHighWater = argBase;
+ return true;
+ }
+ // A script function answering a struct leaves its fields in the argument block, free once the callee copied its own.
+ for (uint8_t i = 0; i < fnCount; i++) {
+ if (fns[i].nameLen != lex.identLen || std::strncmp(fns[i].name, lex.identBeg, lex.identLen) != 0) continue;
+ lex.advance();
+ if (!expect(Tok::LParen, "expected '(' after the function name")) return false;
+ if (!stageScriptArgs(i)) return false;
+ emit({IrOp::CallScript, 0, 0,0,0,0, static_cast(i), nullptr, {}});
+ for (uint8_t f = 0; f < st.fieldCount; f++) {
+ out[f] = alloc();
+ emit({IrOp::LoadCtrl32, out[f], 0,0,0,0, scriptArgOffset(f), nullptr, {}});
+ }
+ return true;
+ }
+ fail("expected a struct value");
+ return false;
+ }
+
+ // Stored once all are evaluated, since a later argument may call through the same block.
+ /// Parse a script call's arguments through its ')' into the arena argument block, a struct taking one word per field.
+ bool stageScriptArgs(uint8_t i) {
+ const uint8_t want = fns[i].params;
+ VReg staged[kMaxScriptArgs] = {};
+ uint8_t got = 0, args = 0;
+ while (lex.kind != Tok::RParen && lex.kind != Tok::End) {
+ if (args > 0 && !expect(Tok::Comma, "expected ',' between arguments")) return false;
+ VReg vals[kMaxStructFields] = {};
+ uint8_t count = 1;
+ // Each argument against its own parameter, as C++ checks a call, so a scalar or another struct cannot pass for the one declared.
+ const bool declared = args < fns[i].argCount;
+ const StructDef* param = declared ? structAt(fns[i].argStruct[args]) : structValueAt();
+ if (param) {
+ if (!parseStructValue(*param, vals)) return false;
+ count = param->fieldCount;
+ } else {
+ if (structValueAt()) { fail("a struct is read one field at a time: write name.field"); return false; }
+ vals[0] = parseExpr();
+ if (failed) return false;
+ }
+ for (uint8_t f = 0; f < count; f++) {
+ if (got < want && got < kMaxScriptArgs) staged[got] = vals[f];
+ else freeTemp(vals[f]);
+ got++;
+ }
+ args++;
+ }
+ if (!expect(Tok::RParen, "expected ')' to close the arguments")) return false;
+ const uint8_t staging = got < want ? got : want;
+ for (uint8_t a = 0; a < staging && a < kMaxScriptArgs; a++) {
+ emit({IrOp::StoreCtrl32, 0, staged[a], 0,0,0, scriptArgOffset(a), nullptr, {}});
+ freeTemp(staged[a]);
+ }
+ if (got != want) {
+ fail(got < want ? "too few arguments for this function" : "too many arguments for this function");
+ return false;
+ }
+ return true;
+ }
+
// The textbook tree-walk register stack, so a multi-call statement fits the register file.
/// Hand out a recycled virtual register, or a fresh one.
VReg alloc() {
@@ -262,9 +482,10 @@ struct Parser {
/// Append an IR op, failing the compile when the program is full or the vreg is out of budget.
void emit(const IrInst& i) { if (!ir.push(i)) fail("script too large"); }
+ // A condition consumes its own && and ||, so one met here stands where a value was wanted.
/// Consume the expected token, or fail with a message.
bool expect(Tok t, const char* msg) {
- if (lex.kind != t) { fail(msg); return false; }
+ if (lex.kind != t) { fail(lex.kind == Tok::AndAnd || lex.kind == Tok::OrOr ? kJoinsConditions : msg); return false; }
lex.advance();
return true;
}
@@ -388,35 +609,125 @@ struct Parser {
return lhs;
}
+ // Explicit, since a silent conversion is 65,536 times off; a shift, not a host call.
+ /// Parse `toFixed(x)` or `toInt(x)`, the cursor on the keyword.
+ VReg parseConversion() {
+ const bool up = atKeyword("toFixed", 7);
+ lex.advance();
+ if (!expect(Tok::LParen, "expected '(' after the conversion")) return 0;
+ VReg v = parseExpr();
+ if (failed) return 0;
+ if (up && exprIsFixed) { fail("this value is already fixed"); return 0; }
+ if (!up && !exprIsFixed) { fail("this value is already a whole number"); return 0; }
+ // A literal is range-checked here, since `toFixed(40000)` would shift past Q16.16.
+ if (up && exprLitConst >= 0) {
+ const int32_t lit = ir.ops[exprLitConst].imm;
+ if (lit < -32768 || lit > 32767)
+ { fail("this number is out of range for a fixed value"); return 0; }
+ }
+ if (!expect(Tok::RParen, "expected ')' to close the conversion")) return 0;
+ VReg dst = alloc();
+ emit({up ? IrOp::Shl : IrOp::Sar, dst, v, 0,0,0, 16, nullptr, {}});
+ freeTemp(v);
+ exprIsFixed = up;
+ return dst;
+ }
+
+ /// Parse what a name reads as a value: a system variable, a struct field, a local, a member, or a call's answer.
+ VReg parseNamePrimary() {
+ // Resolved before locals and members, so a system variable means one thing everywhere.
+ if (const SysVar* sv = sysvars.find(lex.identBeg, lex.identLen)) {
+ lex.advance();
+ if (sv->kind == SysVarKind::Arg) {
+ VReg v = alloc(); // parked at entry, so bring it back for this read
+ emit({IrOp::Reload, v, 0,0,0,0, hostArgSlot(sv->where), nullptr, {}});
+ return v;
+ }
+ VReg v = alloc();
+ // A 4-byte load, since a one-byte read gave a 768-wide wall a width of zero.
+ emit({IrOp::LoadCtrl32, v, 0,0,0,0, sv->where, nullptr, {}});
+ return v;
+ }
+ // A whole struct in a scalar place is read by field: `getRGB(i).r`, `RGB(1, 2, 3).g`.
+ if (const StructDef* st = structCallAt()) {
+ VReg vals[kMaxStructFields] = {};
+ if (!parseStructValue(*st, vals, /*thenField*/ true)) return 0;
+ const int f = parseField(*st);
+ for (uint8_t k = 0; k < st->fieldCount; k++) if (int(k) != f) freeTemp(vals[k]);
+ if (f < 0) return 0;
+ exprIsFixed = false;
+ exprLitConst = -1;
+ return vals[f];
+ }
+ const int li = findLocal(lex.identBeg, lex.identLen);
+ if (li >= 0) {
+ // Into an ordinary temp, so a variable holds a register only for this read.
+ lex.advance();
+ // A struct local's field sits in the slot after the one before it.
+ const StructDef* lst = structAt(locals[li].st);
+ const int f = lst ? parseField(*lst) : 0;
+ if (f < 0) return 0;
+ VReg v = alloc();
+ emit({IrOp::Reload, v, 0,0,0,0, locals[li].slot + f, nullptr, {}});
+ exprIsFixed = (locals[li].type == CtrlType::Fixed);
+ return v;
+ }
+ // One lookup answers both, since a control is a member the UI shows.
+ const int mi = findMember(lex.identBeg, lex.identLen);
+ if (mi >= 0) return parseMemberRead(mi);
+ VReg out = 0;
+ const Builtin* called = table.find(lex.identBeg, lex.identLen);
+ parseCall(&out); // otherwise a call used as an expression must return a value
+ // The builtin's business: uvX and uvY hand back a fixed coordinate, and min follows its arguments.
+ exprIsFixed = called && (called->fixedReturn || (called->sameTypeArgs && argsFixed));
+ exprLitConst = -1;
+ return out;
+ }
+
+ /// Read member `mi`, the cursor past its name: a struct field, an element, or the whole value.
+ VReg parseMemberRead(int mi) {
+ lex.advance();
+ if (const StructDef* mst = structAt(memberStruct[mi])) {
+ const int f = parseField(*mst);
+ if (f < 0) return 0;
+ exprIsFixed = false;
+ return loadMemberField(mi, f);
+ }
+ // An arbitrary index expression, the orthogonality the rest of the language has.
+ if (lex.kind == Tok::LBracket) {
+ if (members[mi].count == 1) { fail("this member is not an array"); return 0; }
+ lex.advance();
+ VReg idx = parseExpr();
+ if (failed) return 0;
+ // An index counts elements, so it is a whole number.
+ if (exprIsFixed) { fail("an array index is a whole number: write toInt(x)"); return 0; }
+ if (!expect(Tok::RBracket, "expected ']' to close an array index")) { freeTemp(idx); return 0; }
+ VReg v = alloc();
+ // The element's type, since leaving the index's made `heat[3] * 0.5` patch the 3.
+ exprIsFixed = (members[mi].type == CtrlType::Fixed);
+ exprLitConst = -1;
+ emit({IrOp::LoadIdx, v, idx, 0, 0, 0,
+ idxPack(members[mi].offset, ctrlWidth(members[mi].type),
+ members[mi].count), nullptr, {}});
+ freeTemp(idx);
+ return v;
+ }
+ if (members[mi].count > 1) { fail("an array needs an index: write name[i]"); return 0; }
+ VReg v = alloc();
+ exprIsFixed = (members[mi].type == CtrlType::Fixed);
+ // One load for every scalar, since a slot already holds what its type promises.
+ emit({IrOp::LoadCtrl32, v, 0,0,0,0, members[mi].offset, nullptr, {}});
+ return v;
+ }
+
// A bare ident reads a declared value, where one followed by `(` is a call.
- /// Parse a primary: a literal, a name, a call, a conversion or a parenthesised expression.
+ /// Parse a primary: a literal, a name, a call, a conversion or a parenthesized expression.
VReg parsePrimary() {
if (failed) return 0;
// Set here, not per branch, so a new kind of primary cannot forget either question.
exprIsFixed = false;
exprLitConst = -1;
- // Explicit, since a silent conversion is 65,536 times off; a shift, not a host call.
- if (lex.kind == Tok::Ident && (atKeyword("toFixed", 7) || atKeyword("toInt", 5))) {
- const bool up = atKeyword("toFixed", 7);
- lex.advance();
- if (!expect(Tok::LParen, "expected '(' after the conversion")) return 0;
- VReg v = parseExpr();
- if (failed) return 0;
- if (up && exprIsFixed) { fail("this value is already fixed"); return 0; }
- if (!up && !exprIsFixed) { fail("this value is already a whole number"); return 0; }
- // A literal is range-checked here, since `toFixed(40000)` would shift past Q16.16.
- if (up && exprLitConst >= 0) {
- const int32_t lit = ir.ops[exprLitConst].imm;
- if (lit < -32768 || lit > 32767)
- { fail("this number is out of range for a fixed value"); return 0; }
- }
- if (!expect(Tok::RParen, "expected ')' to close the conversion")) return 0;
- VReg dst = alloc();
- emit({up ? IrOp::Shl : IrOp::Sar, dst, v, 0,0,0, 16, nullptr, {}});
- freeTemp(v);
- exprIsFixed = up;
- return dst;
- }
+ if (lex.kind == Tok::Ident && (atKeyword("toFixed", 7) || atKeyword("toInt", 5))) return parseConversion();
if (lex.kind == Tok::LParen) { // grouping
lex.advance();
VReg v = parseExpr();
@@ -469,68 +780,8 @@ struct Parser {
lex.advance();
return v;
}
- if (lex.kind == Tok::Ident) {
- // Resolved before locals and members, so a system variable means one thing everywhere.
- if (const SysVar* sv = sysvars.find(lex.identBeg, lex.identLen)) {
- lex.advance();
- if (sv->kind == SysVarKind::Arg) {
- VReg v = alloc(); // parked at entry, so bring it back for this read
- emit({IrOp::Reload, v, 0,0,0,0, hostArgSlot(sv->where), nullptr, {}});
- return v;
- }
- VReg v = alloc();
- // A 4-byte load, since a one-byte read gave a 768-wide wall a width of zero.
- emit({IrOp::LoadCtrl32, v, 0,0,0,0, sv->where, nullptr, {}});
- return v;
- }
- const int li = findLocal(lex.identBeg, lex.identLen);
- if (li >= 0) {
- // Into an ordinary temp, so a variable holds a register only for this read.
- lex.advance();
- VReg v = alloc();
- emit({IrOp::Reload, v, 0,0,0,0, locals[li].slot, nullptr, {}});
- exprIsFixed = (locals[li].type == CtrlType::Fixed);
- return v;
- }
- // One lookup answers both, since a control is a member the UI shows.
- const int mi = findMember(lex.identBeg, lex.identLen);
- if (mi >= 0) {
- lex.advance();
- // An arbitrary index expression, the orthogonality the rest of the language has.
- if (lex.kind == Tok::LBracket) {
- if (members[mi].count == 1) { fail("this member is not an array"); return 0; }
- lex.advance();
- VReg idx = parseExpr();
- if (failed) return 0;
- // An index counts elements, so it is a whole number.
- if (exprIsFixed) { fail("an array index is a whole number: write toInt(x)"); return 0; }
- if (!expect(Tok::RBracket, "expected ']' to close an array index")) { freeTemp(idx); return 0; }
- VReg v = alloc();
- // The element's type, since leaving the index's made `heat[3] * 0.5` patch the 3.
- exprIsFixed = (members[mi].type == CtrlType::Fixed);
- exprLitConst = -1;
- emit({IrOp::LoadIdx, v, idx, 0, 0, 0,
- idxPack(members[mi].offset, ctrlWidth(members[mi].type),
- members[mi].count), nullptr, {}});
- freeTemp(idx);
- return v;
- }
- if (members[mi].count > 1) { fail("an array needs an index: write name[i]"); return 0; }
- VReg v = alloc();
- exprIsFixed = (members[mi].type == CtrlType::Fixed);
- // One load for every scalar, since a slot already holds what its type promises.
- emit({IrOp::LoadCtrl32, v, 0,0,0,0, members[mi].offset, nullptr, {}});
- return v;
- }
- VReg out = 0;
- const Builtin* called = table.find(lex.identBeg, lex.identLen);
- parseCall(&out); // otherwise a call used as an expression must return a value
- // The builtin's business: uvX and uvY hand back a fixed coordinate.
- exprIsFixed = called && called->fixedReturn;
- exprLitConst = -1;
- return out;
- }
- fail("expected a number, a control name, or a function call");
+ if (lex.kind == Tok::Ident) return parseNamePrimary();
+ fail(lex.kind == Tok::Not ? kJoinsConditions : "expected a number, a control name, or a function call");
return 0;
}
@@ -545,29 +796,7 @@ struct Parser {
if (std::strncmp(fns[i].name, lex.identBeg, lex.identLen) != 0) continue;
lex.advance();
if (!expect(Tok::LParen, "expected '(' after the function name")) return;
- // Stored once all are evaluated, since a later one may call through this block.
- const uint8_t want = fns[i].params;
- VReg staged[kMaxScriptArgs] = {};
- uint8_t got = 0;
- while (lex.kind != Tok::RParen && lex.kind != Tok::End) {
- if (got > 0 && !expect(Tok::Comma, "expected ',' between arguments")) return;
- const VReg a = parseExpr();
- if (failed) return;
- if (got < want && got < kMaxScriptArgs) staged[got] = a;
- else freeTemp(a);
- got++;
- }
- if (!expect(Tok::RParen, "expected ')' to close the arguments")) return;
- const uint8_t staging = got < want ? got : want;
- for (uint8_t a = 0; a < staging && a < kMaxScriptArgs; a++) {
- emit({IrOp::StoreCtrl32, 0, staged[a], 0,0,0, scriptArgOffset(a), nullptr, {}});
- freeTemp(staged[a]);
- }
- if (got != want) {
- fail(got < want ? "too few arguments for this function"
- : "too many arguments for this function");
- return;
- }
+ if (!stageScriptArgs(i)) return;
// Refused for a void callee, since the script would read the register's leftovers.
if (resultOut) {
if (fns[i].ret == RetType::Void) {
@@ -594,56 +823,134 @@ struct Parser {
// Parked in consecutive frame slots, so only one argument holds a register at a time.
const uint8_t argBase = slotHighWater;
uint8_t n = 0;
+ if (!parseBuiltinArgs(fn, n)) return;
+ // A builtin answering a struct writes it into its first argument slots, so they stay reserved.
+ if (fn->retStruct) {
+ const StructDef* st = table.findStruct(fn->retStruct, std::strlen(fn->retStruct));
+ if (st && !reserveStructAnswer(argBase, *st)) return;
+ }
+
+ // The arguments live in the frame, so nothing is held in a register across the call.
+ if (resultOut) {
+ if (fn->kind != BuiltinKind::Call || !fn->returns) { fail("this function does not return a value"); return; }
+ // `imm` carries the slot they start at and `b` how many there are.
+ VReg r = alloc();
+ emit({IrOp::Call, r, 0, n, 0, 0, argBase, fn->fn, {}});
+ *resultOut = r;
+ } else {
+ // A statement call, whose result is discarded when it has one.
+ if (fn->kind == BuiltinKind::Call) {
+ VReg r = alloc();
+ emit({IrOp::Call, r, 0, n, 0, 0, argBase, fn->fn, {}});
+ freeTemp(r);
+ } else {
+ // Reloaded into registers, and a wider builtin is refused rather than truncated.
+ if (n > 4) { fail("this function takes too many arguments to inline"); return; }
+ VReg a0 = 0, a1 = 0, a2 = 0, a3 = 0;
+ VReg* slot[4] = {&a0, &a1, &a2, &a3};
+ for (uint8_t i = 0; i < n && i < 4; i++) {
+ *slot[i] = alloc();
+ emit({IrOp::Reload, *slot[i], 0,0,0,0, static_cast(argBase + i), nullptr, {}});
+ }
+ emit({IrOp::Inline, 0, a0, a1, a2, a3, 0, nullptr, nullptr, fn->inlineOp});
+ for (uint8_t i = 0; i < n && i < 4; i++) freeTemp(*slot[i]);
+ }
+ }
+ // Back only to where this call started, since a nested one stages inside its parent's.
+ slotHighWater = argBase;
+ }
+
+ // Parked in consecutive frame slots, so only one argument holds a register at a time.
+ /// Parse a builtin's arguments through its ')' into frame slots from the high-water mark, a struct expanding into its fields; `n` answers how many.
+ bool parseBuiltinArgs(const Builtin* fn, uint8_t& n) {
+ n = 0;
+ // Local, since an argument may itself be a same-type call that would overwrite the member mid-list.
+ bool groupFixed = false;
+ int groupLit = -1;
if (lex.kind != Tok::RParen) {
while (true) {
- if (n >= fn->argc) { fail("too many arguments"); return; }
+ if (n >= fn->argc) { fail("too many arguments"); return false; }
+ // A whole struct fills as many argument positions as it has fields.
+ const StructDef* st = (lex.kind == Tok::Ident && !((fn->byRef >> n) & 1u)) ? structValueAt() : nullptr;
+ if (st) {
+ if (n + st->fieldCount > fn->argc) { fail("too many arguments: this struct fills more positions than remain"); return false; }
+ for (uint8_t f = 0; f < st->fieldCount; f++) {
+ const uint8_t bit = static_cast(1u << (n + f));
+ if ((fn->fixedArgs | fn->byStr | fn->byRef) & bit) {
+ fail("a struct fills whole-number positions only, and this one is not"); return false;
+ }
+ }
+ VReg vals[kMaxStructFields] = {};
+ if (!parseStructValue(*st, vals)) return false;
+ if (slotHighWater + st->fieldCount > kMaxLocals) { fail("too many arguments to hold"); return false; }
+ for (uint8_t f = 0; f < st->fieldCount; f++) {
+ emit({IrOp::Spill, 0, vals[f], 0,0,0, slotHighWater++, nullptr, {}});
+ freeTemp(vals[f]);
+ }
+ n = static_cast(n + st->fieldCount);
+ if (lex.kind == Tok::Comma) { lex.advance(); continue; }
+ break;
+ }
// A string label and a member by name, only where the builtin asks for them.
VReg v = 0;
const bool wantStr = (fn->byStr >> n) & 1u;
if (wantStr && lex.kind != Tok::String) {
- fail("this argument must be a name in quotes"); return;
+ fail("this argument must be a name in quotes"); return false;
}
// Only where wanted, or a pointer's low bits would arrive as a color index.
if (!wantStr && lex.kind == Tok::String) {
- fail("this argument is a number, not a name in quotes"); return;
+ fail("this argument is a number, not a name in quotes"); return false;
}
if (lex.kind == Tok::String) {
// Interned into a pool outliving the compile, not a pointer into the source.
const char* interned = internString(lex.identBeg, lex.identLen);
- if (!interned) { fail("no room for this script's strings"); return; }
+ if (!interned) { fail("no room for this script's strings"); return false; }
v = alloc();
emit({IrOp::ConstPtr, v, 0,0,0,0, 0, nullptr, interned, {}});
lex.advance();
} else if ((fn->byRef >> n) & 1u) {
- if (lex.kind != Tok::Ident) { fail("expected the member this control is bound to"); return; }
+ if (lex.kind != Tok::Ident) { fail("expected the member this control is bound to"); return false; }
const int mi = findMember(lex.identBeg, lex.identLen);
- if (mi < 0) { fail("no member of that name is declared in this class"); return; }
+ if (mi < 0) { fail("no member of that name is declared in this class"); return false; }
// The type must have a widget, and drive one value rather than an array.
if (members[mi].type == CtrlType::Fixed || members[mi].type == CtrlType::Str)
- { fail("a control binds an int, byte or bool member"); return; }
+ { fail("a control binds an int, byte or bool member"); return false; }
if (members[mi].count > 1)
- { fail("a control binds a single member, not an array"); return; }
+ { fail("a control binds a single member, not an array"); return false; }
// Offset and type in one word, since the builtin sees values, not declarations.
v = alloc();
emit({IrOp::Const, v, 0,0,0,0,
members[mi].offset | (int32_t(members[mi].type) << 8), nullptr, {}});
lex.advance();
+ } else if (fn->sameTypeArgs) {
+ // One type across the arguments, a literal adopting it, and the result following it.
+ v = parseExpr();
+ if (failed) return false;
+ if (n == 0) {
+ groupFixed = exprIsFixed;
+ groupLit = exprLitConst;
+ } else {
+ bool outFixed = false;
+ if (!meet(groupFixed, groupLit, exprIsFixed, exprLitConst, outFixed)) return false;
+ groupFixed = outFixed;
+ groupLit = -1;
+ }
} else {
const bool wantFixed = ((fn->fixedArgs >> n) & 1u) != 0;
v = parseExpr();
- if (failed) return;
+ if (failed) return false;
// Against what the builtin declares, since a stray fixed reads 65,536 times off.
if (wantFixed != exprIsFixed) {
bool adopted = false;
if (!wantFixed || !meet(true, -1, exprIsFixed, exprLitConst, adopted)) {
fail(wantFixed ? "this argument is a fixed value: write toFixed(x)"
: "this argument is a whole number: write toInt(x)");
- return;
+ return false;
}
}
}
- if (failed) return;
- if (slotHighWater >= kMaxLocals) { fail("too many arguments to hold"); return; }
+ if (failed) return false;
+ if (slotHighWater >= kMaxLocals) { fail("too many arguments to hold"); return false; }
emit({IrOp::Spill, 0, v, 0,0,0, slotHighWater++, nullptr, {}});
freeTemp(v); // its register is free again at once
n++;
@@ -652,38 +959,35 @@ struct Parser {
}
}
if (slotHighWater > slotsUsed) slotsUsed = slotHighWater;
- if (n != fn->argc) { fail("wrong number of arguments"); return; }
-
- if (!expect(Tok::RParen, "expected ')'")) return;
+ if (n != fn->argc) { fail("wrong number of arguments"); return false; }
+ argsFixed = groupFixed; // read by the caller straight after, before anything else parses
+ return expect(Tok::RParen, "expected ')'");
+ }
- // The arguments live in the frame, so nothing is held in a register across the call.
- if (resultOut) {
- if (fn->kind != BuiltinKind::Call || !fn->returns) { fail("this function does not return a value"); return; }
- // `imm` carries the slot they start at and `b` how many there are.
- VReg r = alloc();
- emit({IrOp::Call, r, 0, n, 0, 0, argBase, fn->fn, {}});
- *resultOut = r;
- } else {
- // A statement call, whose result is discarded when it has one.
- if (fn->kind == BuiltinKind::Call) {
- VReg r = alloc();
- emit({IrOp::Call, r, 0, n, 0, 0, argBase, fn->fn, {}});
- freeTemp(r);
- } else {
- // Reloaded into registers, and a wider builtin is refused rather than truncated.
- if (n > 4) { fail("this function takes too many arguments to inline"); return; }
- VReg a0 = 0, a1 = 0, a2 = 0, a3 = 0;
- VReg* slot[4] = {&a0, &a1, &a2, &a3};
- for (uint8_t i = 0; i < n && i < 4; i++) {
- *slot[i] = alloc();
- emit({IrOp::Reload, *slot[i], 0,0,0,0, static_cast(argBase + i), nullptr, {}});
- }
- emit({IrOp::Inline, 0, a0, a1, a2, a3, 0, nullptr, nullptr, fn->inlineOp});
- for (uint8_t i = 0; i < n && i < 4; i++) freeTemp(*slot[i]);
- }
- }
- // Back only to where this call started, since a nested one stages inside its parent's.
- slotHighWater = argBase;
+ // Its fields pack at their own width from a 4-byte boundary, as an array's elements do, and start at zero.
+ /// Parse a struct member, `RGB c;`, the type name already consumed.
+ void parseStructDecl(const StructDef& st) {
+ if (lex.kind != Tok::Ident) { fail("expected a member name after the type"); return; }
+ const char* name = lex.identBeg; size_t nameLen = lex.identLen;
+ if (nameLen >= kMaxControlName) { fail("member name too long"); return; }
+ if (sysvars.find(name, nameLen)) { fail("name is a system variable"); return; }
+ if (findMember(name, nameLen) >= 0) { fail("duplicate member name"); return; }
+ if (table.find(name, nameLen)) { fail("member name shadows a built-in function"); return; }
+ if (isReservedWord(name, nameLen)) { fail("member name is a reserved word"); return; }
+ if (table.findStruct(name, nameLen)) { fail("that name is a type"); return; }
+ lex.advance();
+ if (lex.kind == Tok::Assign) { fail("a struct member starts at zero: set its fields in a function"); return; }
+ if (!expect(Tok::Semicolon, "expected ';' after the member declaration")) return;
+ if (memberCount >= kMaxCtrls) { fail("too many members"); return; }
+ uint16_t at = memberBytes;
+ if ((at % 4) != 0) at = uint16_t(at + (4 - at % 4));
+ const uint16_t need = uint16_t(st.fieldCount) * ctrlWidth(st.type);
+ if (at + need > kCtrlBytes) { fail("the class declares more member data than the arena holds"); return; }
+ members[memberCount] = {name, 0, 0, 0, static_cast(nameLen), st.type,
+ static_cast(at), st.fieldCount};
+ memberStruct[memberCount] = structIndex(&st);
+ memberBytes = static_cast(at + need);
+ memberCount++;
}
// The leading type keyword is already consumed by the caller.
@@ -699,6 +1003,7 @@ struct Parser {
if (table.find(name, nameLen)) { fail("member name shadows a built-in function"); return; }
// A reserved word resolves first, so such a member could be declared and never read.
if (isReservedWord(name, nameLen)) { fail("member name is a reserved word"); return; }
+ if (table.findStruct(name, nameLen)) { fail("that name is a type"); return; }
lex.advance();
// The length is a literal, since the arena is sized at compile time.
uint8_t count = 1;
@@ -821,6 +1126,18 @@ struct Parser {
return false;
}
+ /// The struct type the current token names, or null.
+ const StructDef* structTypeAt() const {
+ return lex.kind == Tok::Ident ? table.findStruct(lex.identBeg, lex.identLen) : nullptr;
+ }
+ /// Whether a struct declaration starts here: a type name followed by a variable name.
+ bool structDeclAt() const {
+ if (!structTypeAt()) return false;
+ Lexer probe = lex;
+ probe.advance();
+ return probe.kind == Tok::Ident;
+ }
+
// Matched by text rather than lexed, since the set is tiny and a length check is exact.
/// Whether the current token is exactly this keyword.
bool atKeyword(const char* kw, size_t len) const {
@@ -994,10 +1311,12 @@ struct Parser {
// A system variable may not be assigned, since the engine rewrites it before every call.
/// Parse an assignment, with the name already consumed.
bool parseAssignment(const char* name, size_t nameLen) {
+ if (lex.kind == Tok::Dot) return parseFieldAssignment(name, nameLen);
// Resolved before the '=', since an index sits between the name and the operator.
if (lex.kind == Tok::LBracket) {
const int ai = findMember(name, nameLen);
if (ai < 0) { fail("no member of that name is declared in this class"); return false; }
+ if (memberStruct[ai]) { fail(kOneFieldAtATime); return false; }
if (members[ai].count == 1) { fail("this member is not an array"); return false; }
lex.advance();
VReg idx = parseExpr();
@@ -1030,6 +1349,13 @@ struct Parser {
// The destination first, so a refused target reports the name the script wrote.
const int li = findLocal(name, nameLen);
const int mi = li >= 0 ? -1 : findMember(name, nameLen);
+ // A whole struct copies field by field.
+ if (const StructDef* st = li >= 0 ? structAt(locals[li].st) : (mi >= 0 ? structAt(memberStruct[mi]) : nullptr)) {
+ VReg vals[kMaxStructFields] = {};
+ if (!parseStructValue(*st, vals)) return false;
+ storeStruct(*st, li, mi, vals);
+ return expect(Tok::Semicolon, "expected ';' after an assignment");
+ }
if (mi >= 0 && members[mi].count > 1) {
fail("assign one element: name[i] = value");
return false;
@@ -1075,68 +1401,216 @@ struct Parser {
return expect(Tok::Semicolon, "expected ';' after an assignment");
}
+ /// Store a struct value's registers into a local's slots or a member's bytes, freeing them.
+ void storeStruct(const StructDef& st, int li, int mi, VReg vals[kMaxStructFields]) {
+ for (uint8_t f = 0; f < st.fieldCount; f++) {
+ if (li >= 0) emit({IrOp::Spill, 0, vals[f], 0,0,0, locals[li].slot + f, nullptr, {}});
+ else emit({storeOpFor(members[mi].type), 0, vals[f], 0,0,0, memberFieldOffset(mi, f), nullptr, {}});
+ freeTemp(vals[f]);
+ }
+ }
+
+ /// Parse `name.field = value;`, the name consumed and the '.' current.
+ bool parseFieldAssignment(const char* name, size_t nameLen) {
+ const int li = findLocal(name, nameLen);
+ const int mi = li >= 0 ? -1 : findMember(name, nameLen);
+ const StructDef* st = li >= 0 ? structAt(locals[li].st) : (mi >= 0 ? structAt(memberStruct[mi]) : nullptr);
+ if (!st) { fail(li < 0 && mi < 0 ? "not declared: add it to the class body" : "only a struct has fields"); return false; }
+ const int f = parseField(*st);
+ if (f < 0) return false;
+ if (!expect(Tok::Assign, "expected '=' in an assignment")) return false;
+ VReg v = parseExpr();
+ if (failed) return false;
+ if (exprIsFixed) { fail("a struct field is a whole number: write toInt(x)"); return false; }
+ if (exprLitConst >= 0 && !fitsInType(ir.ops[exprLitConst].imm, st->type))
+ { fail("this number is out of range for that field"); return false; }
+ if (li >= 0) {
+ narrowToType(v, st->type); // a byte field wraps at 255, as a byte local does
+ emit({IrOp::Spill, 0, v, 0,0,0, locals[li].slot + f, nullptr, {}});
+ } else {
+ emit({storeOpFor(members[mi].type), 0, v, 0,0,0, memberFieldOffset(mi, f), nullptr, {}});
+ }
+ freeTemp(v);
+ return expect(Tok::Semicolon, "expected ';' after an assignment");
+ }
+
// A bool holds 0..255, so a multiple of 256 truncates to false: the IR has no select.
/// The store op a member's type needs.
static IrOp storeOpFor(CtrlType type) {
return ctrlWidth(type) == 1 ? IrOp::StoreCtrl : IrOp::StoreCtrl32;
}
- // Six comparisons onto two signed branch ops, each emitting the negation the script wrote.
- /// Parse an `if` with an optional `else`, and emit it.
- bool parseIf() {
- lex.advance(); // `if`
- if (!expect(Tok::LParen, "expected '(' after if")) return false;
- VReg a = parseExpr();
- if (failed) return false;
- const Tok cmp = lex.kind;
- if (cmp != Tok::Less && cmp != Tok::LessEq && cmp != Tok::Greater &&
- cmp != Tok::GreaterEq && cmp != Tok::EqEq && cmp != Tok::NotEq) {
- freeTemp(a);
- fail("expected a comparison: <, <=, >, >=, == or !=");
- return false;
- }
- const bool aFixed = exprIsFixed;
- const int aLit = exprLitConst;
- lex.advance();
- VReg b = parseExpr();
- if (failed) { freeTemp(a); return false; }
- // Same wall as the operators, since Q16.16 preserves order but not magnitude.
- bool cmpFixed = false;
- if (!meet(aFixed, aLit, exprIsFixed, exprLitConst, cmpFixed))
- { freeTemp(b); freeTemp(a); return false; }
- if (!expect(Tok::RParen, "expected ')' to close the if condition")) { freeTemp(b); freeTemp(a); return false; }
- if (!expect(Tok::LBrace, "expected '{': an if body is braced")) { freeTemp(b); freeTemp(a); return false; }
+ /// A fresh branch label, or -1 having failed when the script has used them all.
+ int newLabel() {
+ if (nextLabel >= kIrLabels) { fail("too many branches in one script"); return -1; }
+ return nextLabel++;
+ }
- // Two labels at most: one to skip the then-block, one to skip the else-block.
- if (nextLabel + 2 > kIrLabels) { fail("too many branches in one script"); return false; }
- const uint8_t lElse = nextLabel++;
+ // The jump idiom, `z >= z` with z zero, since the IR has no unconditional branch.
+ /// Branch to a label always.
+ void emitJump(uint8_t label) {
+ VReg z = alloc();
+ emit({IrOp::Const, z, 0,0,0,0, 0, nullptr, {}});
+ emit({IrOp::BranchGe, 0, z, z, 0,0, label, nullptr, {}});
+ freeTemp(z);
+ }
- // Emit the branch that SKIPS the then-block, which is the NEGATION of the written test.
+ // Six comparisons onto two signed branch ops, each emitting the negation the script wrote.
+ /// Branch to `lFalse` unless `a cmp b` holds, freeing both registers.
+ void emitBranchUnless(Tok cmp, VReg a, VReg b, uint8_t lFalse) {
switch (cmp) {
// Signed, since an unsigned compare reads a negative as a huge value.
- case Tok::Less: emit({IrOp::BranchGeS, 0, a, b, 0,0, lElse, nullptr, {}}); break;
+ case Tok::Less: emit({IrOp::BranchGeS, 0, a, b, 0,0, lFalse, nullptr, {}}); break;
// !(a >= b) is a < b, which needs the strict pair below.
- case Tok::GreaterEq: emitStrictLess(a, b, lElse); break;
+ case Tok::GreaterEq: emitStrictLess(a, b, lFalse); break;
// !(a > b) is a <= b, i.e. b >= a.
- case Tok::Greater: emit({IrOp::BranchGeS, 0, b, a, 0,0, lElse, nullptr, {}}); break;
+ case Tok::Greater: emit({IrOp::BranchGeS, 0, b, a, 0,0, lFalse, nullptr, {}}); break;
// !(a <= b) is a > b, i.e. b < a.
- case Tok::LessEq: emitStrictLess(b, a, lElse); break;
- case Tok::EqEq: emit({IrOp::BranchNe, 0, a, b, 0,0, lElse, nullptr, {}}); break;
+ case Tok::LessEq: emitStrictLess(b, a, lFalse); break;
+ case Tok::EqEq: emit({IrOp::BranchNe, 0, a, b, 0,0, lFalse, nullptr, {}}); break;
// There is no BranchEq, so this is the one case needing two branches.
case Tok::NotEq: {
- if (nextLabel >= kIrLabels) { freeTemp(b); freeTemp(a); fail("too many branches in one script"); return false; }
+ if (nextLabel >= kIrLabels) { freeTemp(b); freeTemp(a); fail("too many branches in one script"); return; }
const uint8_t lBody = nextLabel++;
emit({IrOp::BranchNe, 0, a, b, 0,0, lBody, nullptr, {}});
- VReg z = alloc();
- emit({IrOp::Const, z, 0,0,0,0, 0, nullptr, {}});
- emit({IrOp::BranchGe, 0, z, z, 0,0, lElse, nullptr, {}}); // z >= z: always taken
- freeTemp(z);
+ emitJump(lFalse);
emit({IrOp::Label, 0, 0,0,0,0, lBody, nullptr, {}});
break;
}
default: break;
}
freeTemp(b); freeTemp(a);
+ }
+
+ /// Whether a token compares two values.
+ static bool isComparison(Tok t) {
+ return t == Tok::Less || t == Tok::LessEq || t == Tok::Greater || t == Tok::GreaterEq ||
+ t == Tok::EqEq || t == Tok::NotEq;
+ }
+
+ // Peeks without consuming, to the ')' that closes the condition or group the cursor is in.
+ /// Whether a `||` follows at this nesting level.
+ bool orFollows() const {
+ Lexer probe = lex;
+ int depth = 0;
+ while (probe.kind != Tok::End && probe.kind != Tok::Error) {
+ if (probe.kind == Tok::LParen) depth++;
+ else if (probe.kind == Tok::RParen) { if (depth == 0) return false; depth--; }
+ else if (probe.kind == Tok::OrOr && depth == 0) return true;
+ probe.advance();
+ }
+ return false;
+ }
+
+ // `(a + b) < c` opens with a parenthesized expression, `(a < b) && c` with a parenthesized condition.
+ /// Whether the '(' at the cursor opens a condition rather than an arithmetic group.
+ bool groupIsCondition() const {
+ Lexer probe = lex;
+ probe.advance(); // past the '('
+ int depth = 0;
+ while (probe.kind != Tok::End && probe.kind != Tok::Error) {
+ if (probe.kind == Tok::LParen) depth++;
+ else if (probe.kind == Tok::RParen) { if (depth == 0) return false; depth--; }
+ else if (depth == 0 && (isComparison(probe.kind) || probe.kind == Tok::AndAnd ||
+ probe.kind == Tok::OrOr || probe.kind == Tok::Not)) return true;
+ probe.advance();
+ }
+ return false;
+ }
+
+ // Jumping code, as C compiles a condition: every test branches to `lFalse` when it fails, so `&&` needs no label and `||` one per term.
+ /// Parse a condition, branching to `lFalse` when it is false and falling through when it is true.
+ bool parseCondition(uint8_t lFalse) {
+ if (!orFollows()) return parseConjunction(lFalse);
+ const int lTrue = newLabel();
+ if (lTrue < 0) return false;
+ for (;;) {
+ const int lNext = newLabel();
+ if (lNext < 0) return false;
+ if (!parseConjunction(static_cast(lNext))) return false;
+ emitJump(static_cast(lTrue)); // this term held: the whole condition holds
+ emit({IrOp::Label, 0, 0,0,0,0, lNext, nullptr, {}});
+ if (!expect(Tok::OrOr, "expected '||'")) return false;
+ if (!orFollows()) break;
+ }
+ if (!parseConjunction(lFalse)) return false; // the last term decides
+ emit({IrOp::Label, 0, 0,0,0,0, lTrue, nullptr, {}});
+ return true;
+ }
+
+ /// Parse terms joined by `&&`, each branching to `lFalse` when it fails.
+ bool parseConjunction(uint8_t lFalse) {
+ if (!parseNegation(lFalse)) return false;
+ while (!failed && lex.kind == Tok::AndAnd) {
+ lex.advance();
+ if (!parseNegation(lFalse)) return false;
+ }
+ return !failed;
+ }
+
+ /// Parse a test with any number of leading `!`, each one swapping where it branches.
+ // As in C, `!` negates what follows it rather than a comparison after that, so `!a < b` would compare !a: refused, so it is read neither way.
+ bool parseNegation(uint8_t lFalse) {
+ if (lex.kind != Tok::Not) return parseCondPrimary(lFalse);
+ lex.advance();
+ if (lex.kind != Tok::Not && !(lex.kind == Tok::LParen && groupIsCondition())) {
+ // A negated value holds when the value is zero.
+ VReg a = parseExpr();
+ if (failed) return false;
+ if (isComparison(lex.kind)) { freeTemp(a); fail(kNegatesWhatFollows); return false; }
+ VReg z = alloc();
+ emit({IrOp::Const, z, 0,0,0,0, 0, nullptr, {}});
+ emitBranchUnless(Tok::EqEq, a, z, lFalse);
+ return !failed;
+ }
+ const int lHeld = newLabel(); // where the inner test goes when it fails, which is when this one holds
+ if (lHeld < 0) return false;
+ if (!parseNegation(static_cast(lHeld))) return false;
+ if (isComparison(lex.kind)) { fail(kNegatesWhatFollows); return false; }
+ emitJump(lFalse); // the inner test held, so this one fails
+ emit({IrOp::Label, 0, 0,0,0,0, lHeld, nullptr, {}});
+ return true;
+ }
+
+ // A bare value is true when it is not zero, as C reads one, so `if (done)` and `if (!done)` read naturally.
+ /// Parse one test: a parenthesized condition, a comparison, or a value.
+ bool parseCondPrimary(uint8_t lFalse) {
+ if (lex.kind == Tok::LParen && groupIsCondition()) {
+ lex.advance();
+ if (!parseCondition(lFalse)) return false;
+ return expect(Tok::RParen, "expected ')' to close the condition");
+ }
+ VReg a = parseExpr();
+ if (failed) return false;
+ const Tok cmp = lex.kind;
+ if (!isComparison(cmp)) {
+ VReg z = alloc();
+ emit({IrOp::Const, z, 0,0,0,0, 0, nullptr, {}});
+ emitBranchUnless(Tok::NotEq, a, z, lFalse);
+ return !failed;
+ }
+ const bool aFixed = exprIsFixed;
+ const int aLit = exprLitConst;
+ lex.advance();
+ VReg b = parseExpr();
+ if (failed) { freeTemp(a); return false; }
+ // Same wall as the operators, since Q16.16 preserves order but not magnitude.
+ bool cmpFixed = false;
+ if (!meet(aFixed, aLit, exprIsFixed, exprLitConst, cmpFixed)) { freeTemp(b); freeTemp(a); return false; }
+ emitBranchUnless(cmp, a, b, lFalse);
+ return !failed;
+ }
+
+ /// Parse an `if` with an optional `else`, and emit it.
+ bool parseIf() {
+ lex.advance(); // `if`
+ if (!expect(Tok::LParen, "expected '(' after if")) return false;
+ // One label to skip the then-block, taken before the condition so a single comparison emits what it always did.
+ if (nextLabel + 2 > kIrLabels) { fail("too many branches in one script"); return false; }
+ const uint8_t lElse = nextLabel++;
+ if (!parseCondition(lElse)) return false;
+ if (!expect(Tok::RParen, "expected ')' to close the if condition")) return false;
+ if (!expect(Tok::LBrace, "expected '{': an if body is braced")) return false;
// A braced body is a scope, so a local dies at the '}' and its slot is handed back.
const uint8_t thenLocal = localCount, thenSlot = slotHighWater;
@@ -1207,6 +1681,7 @@ struct Parser {
const size_t varLen = lex.identLen;
// The same collisions a loop counter checks for, and for the same reasons.
if (isReservedWord(varName, varLen)) { fail("that name is a reserved word"); return false; }
+ if (table.findStruct(varName, varLen)) { fail("that name is a type"); return false; }
// A local named for a builtin would shadow it for the rest of the function.
if (table.find(varName, varLen)) { fail("that name shadows a built-in function"); return false; }
if (sysvars.find(varName, varLen)) { fail("name is a system variable"); return false; }
@@ -1244,8 +1719,48 @@ struct Parser {
return expect(Tok::Semicolon, "expected ';' after a variable declaration");
}
+ // One slot per field for the rest of the block, from a struct value or zero.
+ /// Parse a struct local, `RGB c = value;` or `RGB c;`.
+ bool parseStructLocalDecl() {
+ const StructDef& st = *structTypeAt();
+ lex.advance();
+ const char* varName = lex.identBeg;
+ const size_t varLen = lex.identLen;
+ if (isReservedWord(varName, varLen)) { fail("that name is a reserved word"); return false; }
+ if (table.findStruct(varName, varLen)) { fail("that name is a type"); return false; }
+ if (table.find(varName, varLen)) { fail("that name shadows a built-in function"); return false; }
+ if (sysvars.find(varName, varLen)) { fail("name is a system variable"); return false; }
+ if (findLocal(varName, varLen) >= 0) { fail("that name is already in use here"); return false; }
+ if (findMember(varName, varLen) >= 0) { fail("a member of that name is declared"); return false; }
+ if (localCount >= kMaxLocals || slotHighWater + st.fieldCount > kMaxLocals) {
+ fail("too many variables in this function: a struct takes one slot per field");
+ return false;
+ }
+ lex.advance();
+ VReg vals[kMaxStructFields] = {};
+ if (lex.kind == Tok::Semicolon) {
+ for (uint8_t f = 0; f < st.fieldCount; f++) {
+ vals[f] = alloc();
+ emit({IrOp::Const, vals[f], 0,0,0,0, 0, nullptr, {}});
+ }
+ } else {
+ if (!expect(Tok::Assign, "expected '=' or ';' after the variable name")) return false;
+ if (!parseStructValue(st, vals)) return false;
+ }
+ const uint8_t slot = slotHighWater;
+ slotHighWater = static_cast(slotHighWater + st.fieldCount);
+ if (slotHighWater > slotsUsed) slotsUsed = slotHighWater;
+ for (uint8_t f = 0; f < st.fieldCount; f++) {
+ emit({IrOp::Spill, 0, vals[f], 0,0,0, slot + f, nullptr, {}});
+ freeTemp(vals[f]);
+ }
+ locals[localCount++] = {varName, varLen, slot, st.type, structIndex(&st)};
+ return expect(Tok::Semicolon, "expected ';' after a variable declaration");
+ }
+
/// Parse one statement: a declaration, a for, an if, a return, an assignment or a call.
bool parseStatement() {
+ if (structDeclAt()) return parseStructLocalDecl();
if (atTypeKeyword()) return parseLocalDecl();
if (atKeyword("for", 3)) return parseFor();
if (atKeyword("if", 2)) return parseIf();
@@ -1256,8 +1771,8 @@ struct Parser {
const size_t nameLen = lex.identLen;
Lexer save = lex;
lex.advance();
- // `name =` and `name[` open an assignment, where `name(` opens a call.
- if (lex.kind == Tok::Assign || lex.kind == Tok::LBracket) return parseAssignment(name, nameLen);
+ // `name =`, `name[` and `name.` open an assignment, where `name(` opens a call.
+ if (lex.kind == Tok::Assign || lex.kind == Tok::LBracket || lex.kind == Tok::Dot) return parseAssignment(name, nameLen);
lex = save;
parseCall(nullptr);
if (failed) return false;
@@ -1277,6 +1792,17 @@ struct Parser {
}
// A value from a void function has nowhere to go.
if (curRet == RetType::Void) { fail("a void function returns no value"); return false; }
+ // A struct's fields go into the argument block, which the caller reads after the call.
+ if (curRet == RetType::Struct) {
+ VReg vals[kMaxStructFields] = {};
+ if (!parseStructValue(*curRetStruct, vals)) return false;
+ for (uint8_t f = 0; f < curRetStruct->fieldCount; f++) {
+ emit({IrOp::StoreCtrl32, 0, vals[f], 0,0,0, scriptArgOffset(f), nullptr, {}});
+ freeTemp(vals[f]);
+ }
+ emit({IrOp::Ret, 0, 0,0,0,0, 0, nullptr, {}});
+ return expect(Tok::Semicolon, "expected ';'");
+ }
VReg v = 0;
if (lex.kind == Tok::String) {
if (curRet != RetType::Str) { fail("this function returns an int, not a string"); return false; }
@@ -1299,7 +1825,7 @@ struct Parser {
// Parsed before anything is emitted, since a later insertion would shift every index.
/// Parse a function's parameter list, answering how many it declares.
- bool parseParams(uint8_t* paramsOut) {
+ bool parseParams(uint8_t* paramsOut, uint8_t* argCountOut, uint8_t argStructOut[moonlive::kMaxScriptArgs]) {
if (!expect(Tok::LParen, "expected '(' after the function name")) return false;
// Cleared first, since a leftover would push the parameters up out of their slots.
localCount = 0; slotHighWater = 0;
@@ -1307,14 +1833,17 @@ struct Parser {
while (lex.kind != Tok::RParen && lex.kind != Tok::End) {
if (params > 0 && !expect(Tok::Comma, "expected ',' between parameters")) return false;
// A parameter is declared like a variable, so every type means what it means elsewhere.
- if (!atTypeKeyword()) { fail("expected a parameter type"); return false; }
- const CtrlType t = currentType();
+ const StructDef* pst = structTypeAt();
+ if (!pst && !atTypeKeyword()) { fail("expected a parameter type"); return false; }
+ const CtrlType t = pst ? pst->type : currentType();
+ const uint8_t words = pst ? pst->fieldCount : 1;
lex.advance();
if (lex.kind != Tok::Ident) { fail("expected a parameter name"); return false; }
// A parameter is a local, so a name refused inside the body is refused here.
if (isReservedWord(lex.identBeg, lex.identLen)) {
fail("that name is a reserved word"); return false;
}
+ if (table.findStruct(lex.identBeg, lex.identLen)) { fail("that name is a type"); return false; }
if (table.find(lex.identBeg, lex.identLen)) {
fail("that name shadows a built-in function"); return false;
}
@@ -1328,15 +1857,17 @@ struct Parser {
fail("a member of that name is declared"); return false;
}
if (localCount >= kMaxLocals) { fail("too many locals"); return false; }
- // Bounded by the arena block, since a fifth load reads past its end.
- if (params >= moonlive::kMaxScriptArgs) {
- fail("a function takes at most 4 parameters");
+ // Bounded by the arena block, since a further load reads past its end.
+ if (params + words > moonlive::kMaxScriptArgs) {
+ fail("parameters fill at most 8 words, a struct taking one per field");
return false;
}
- locals[localCount++] = {lex.identBeg, lex.identLen, slotHighWater, t};
- slotHighWater++;
+ locals[localCount++] = {lex.identBeg, lex.identLen, slotHighWater, t, structIndex(pst)};
+ argStructOut[(*argCountOut)++] = structIndex(pst); // a struct fills at least one word, so the count stays within the block
+ for (uint8_t w = 0; w < words; w++) paramWordType[params + w] = t;
+ slotHighWater = static_cast(slotHighWater + words);
if (slotHighWater > slotsUsed) slotsUsed = slotHighWater;
- params++;
+ params = static_cast(params + words);
lex.advance();
}
if (!expect(Tok::RParen, "expected ')' to close the parameter list")) return false;
@@ -1369,7 +1900,8 @@ struct Parser {
if (!expect(Tok::LBrace, "expected '{' to open the class body")) return false;
// Two tokens of lookahead, since a member and a typed function open with the same one.
- while (!failed && atTypeKeyword()) {
+ while (!failed && (atTypeKeyword() || structTypeAt())) {
+ const StructDef* memberSt = structTypeAt();
const CtrlType ty = currentType();
Lexer save = lex;
lex.advance(); // past the type
@@ -1379,18 +1911,20 @@ struct Parser {
return probe.kind == Tok::LParen;
}();
if (isFn) { lex = save; break; } // a function: leave it for the loop below
- parseDecl(ty);
+ if (memberSt) parseStructDecl(*memberSt);
+ else parseDecl(ty);
}
if (failed) return false;
bool any = false;
while (!failed && lex.kind != Tok::RBrace && lex.kind != Tok::End) {
// Declared rather than implicit, so the host can rely on what a function hands back.
- if (!atRetKeyword()) {
- fail("a function declares what it returns: void, int or string");
+ const StructDef* retSt = structTypeAt();
+ if (!atRetKeyword() && !retSt) {
+ fail("a function returns void, int, string or a struct type");
return false;
}
- const RetType ret = currentRet();
+ const RetType ret = retSt ? RetType::Struct : currentRet();
lex.advance(); // past the return type
if (lex.kind != Tok::Ident) { fail("expected a function name after its return type"); return false; }
if (fnCount >= kMaxEntryPoints) { fail("too many functions in one class"); return false; }
@@ -1399,11 +1933,23 @@ struct Parser {
// The name is captured before the signature, since parsing moves the lexer.
const char* const fnName = lex.identBeg;
const uint8_t fnNameLen = static_cast(lex.identLen);
+ // A builtin is found before a script function, so one of the same name would never be called.
+ if (isReservedWord(fnName, fnNameLen)) { fail("that name is a reserved word"); return false; }
+ if (table.findStruct(fnName, fnNameLen)) { fail("that name is a type"); return false; }
+ if (table.find(fnName, fnNameLen)) { fail("that name shadows a built-in function"); return false; }
lex.advance(); // the function name
uint8_t params = 0;
- if (!parseParams(¶ms)) return false;
- fns[fnCount] = {fnName, fnNameLen, static_cast(ir.count), ret, params};
+ FnMark& mark = fns[fnCount];
+ mark = FnMark{};
+ if (!parseParams(¶ms, &mark.argCount, mark.argStruct)) return false;
+ mark.name = fnName;
+ mark.nameLen = fnNameLen;
+ mark.irStart = static_cast(ir.count);
+ mark.ret = ret;
+ mark.params = params;
+ mark.retStruct = structIndex(retSt);
curRet = ret; // every return below is checked against it
+ curRetStruct = retSt;
// The IR carries the start INDEX; the lowering turns it into a byte offset.
ir.fnIrStart[fnCount] = static_cast(ir.count);
ir.fnCount = static_cast(fnCount + 1);
@@ -1416,7 +1962,7 @@ struct Parser {
const VReg v = alloc();
emit({IrOp::LoadCtrl32, v, 0,0,0,0, scriptArgOffset(a), nullptr, {}});
// Narrowed here, since the callee knows the types and serves every caller once.
- narrowToType(v, locals[a].type);
+ narrowToType(v, paramWordType[a]);
emit({IrOp::Spill, 0, v, 0,0,0, a, nullptr, {}});
freeTemp(v);
}
@@ -1450,7 +1996,7 @@ CompileResult compileSource(const char* source, const BuiltinTable& table,
if (!source) { r.error = "no source"; return r; }
if (!out || cap == 0) { r.error = "no code buffer"; return r; }
- // Sized from the token count against an empirical ratio, since a worst case gets no memory.
+ // Sized from the token count against the shipped scripts' ratio; a denser script grows the array as it goes.
const uint32_t tokens = countTokens(source);
IrProgram ir;
// The margin covers a program's fixed overhead, so one token cannot round down to nothing.
@@ -1477,6 +2023,7 @@ CompileResult compileSource(const char* source, const BuiltinTable& table,
case LowerRefusal::NullCall: r.error = "codegen failed: a builtin has no function on this target"; break;
case LowerRefusal::AsmOverflow: r.error = "codegen failed: assembler overflow (branch range, slot, or immediate)"; break;
case LowerRefusal::OverCap: r.error = "codegen failed: code larger than its buffer"; break;
+ case LowerRefusal::NoMemory: r.error = "codegen failed: out of memory for the assembler"; break;
default: r.error = kCodegenFailed; break;
}
return r;
diff --git a/src/core/moonlive/MoonLiveCompiler.h b/src/core/moonlive/MoonLiveCompiler.h
index cc6deaf26..368b7386a 100644
--- a/src/core/moonlive/MoonLiveCompiler.h
+++ b/src/core/moonlive/MoonLiveCompiler.h
@@ -25,7 +25,7 @@ inline constexpr const char* kSpillRefused = "codegen failed: too many live val
// A symbol table: a binding asks for an entry by name, so a script's role follows from what it defined.
/// What a function hands back, where `void` is the default an acting entry point declares.
-enum class RetType : uint8_t { Void, Int, Str };
+enum class RetType : uint8_t { Void, Int, Str, Struct };
struct EntryPoint {
const char* name = nullptr; ///< into the source, or the engine's own copy after compile
diff --git a/src/core/moonlive/MoonLiveIr.h b/src/core/moonlive/MoonLiveIr.h
index 5e54f74a1..a2fc7c39b 100644
--- a/src/core/moonlive/MoonLiveIr.h
+++ b/src/core/moonlive/MoonLiveIr.h
@@ -1,6 +1,7 @@
#pragma once
#include
+#include
#include
#include "core/moonlive/MoonLiveBuiltins.h" // InlineOp (a neutral opcode tag)
#include "platform/platform.h" // alloc/free, since the op array is script-sized
@@ -35,7 +36,7 @@ static constexpr uint8_t kMaxVRegs = 32;
// A sanity bound rather than the working limit, so a runaway source fails with a diagnostic.
static constexpr uint16_t kMaxIrOps = 4096;
-// Measured at 0.75 ops per token and never above 0.85, so 1 keeps a real margin.
+// The shipped scripts measure up to 0.91 ops per token; structs and `&&` reach 1.6, and grow the array rather than raise every script's reservation.
static constexpr uint16_t kIrOpsPerToken = 1;
// The op set, in three-address form: a destination plus up to three source operands.
@@ -137,12 +138,12 @@ constexpr uint8_t idxCount(int32_t p) { return uint8_t((p >> 16) & 0xff); }
// Two per `for`, counted program-wide, so this bounds total loops rather than nesting depth.
/// Branch targets one IR program may use.
-static constexpr uint8_t kIrLabels = 40;
+static constexpr uint8_t kIrLabels = 96;
-// THE COST IS STACK: a label is a two-byte code position, since no program passes the 16 KB code cap, so 96 of them take what 48 four-byte ones did.
-/// Labels and fixups an assembler's tables hold.
-static constexpr uint8_t kAsmLabels = 96;
-static constexpr uint8_t kAsmFixups = 96;
+// On the heap for one compile; a label is a two-byte code position and a fixup four bytes, under the 16 KB code cap.
+/// Labels and fixups an assembler's tables hold, the most an eight-bit index addresses.
+static constexpr uint8_t kAsmLabels = 255;
+static constexpr uint8_t kAsmFixups = 255;
/// Where a label is bound in the code, or kUnbound before it is.
using CodePos = uint16_t;
static constexpr CodePos kUnbound = 0xFFFF;
@@ -212,9 +213,24 @@ struct IrProgram {
return ops != nullptr;
}
- /// Append one op, returning false when it names a register outside the budget.
+ // By half, as a dynamic array grows, so the reservation can be the typical script's size and a dense one still fits.
+ /// Enlarge the op array, keeping its contents, up to kMaxIrOps; false at the bound or out of memory.
+ bool grow() {
+ if (!ops || cap >= kMaxIrOps) return false;
+ const uint32_t want = uint32_t(cap) + cap / 2 + 8;
+ const uint16_t n = static_cast(want > kMaxIrOps ? kMaxIrOps : want);
+ auto* bigger = static_cast(platform::alloc(sizeof(IrInst) * n));
+ if (!bigger) return false;
+ std::memcpy(bigger, ops, sizeof(IrInst) * count);
+ platform::free(ops);
+ ops = bigger;
+ cap = n;
+ return true;
+ }
+
+ /// Append one op, returning false when it names a register outside the budget or the array cannot grow.
bool push(const IrInst& i) {
- if (!ops || count >= cap) return false;
+ if (!ops || (count >= cap && !grow())) return false;
// Opcode-specific, since not every operand field holds a vreg: a Call's `b` is a count.
if (i.dst >= kMaxVRegs) return false;
if (i.op == IrOp::Call) {
diff --git a/src/core/moonlive/MoonLiveScript.h b/src/core/moonlive/MoonLiveScript.h
index d558522b9..30d0bd7f7 100644
--- a/src/core/moonlive/MoonLiveScript.h
+++ b/src/core/moonlive/MoonLiveScript.h
@@ -37,6 +37,9 @@ class MoonLiveScript {
/// Let a binding attach its own sinks around the script's defineControls(), such as a pool it sizes.
void setDefineHook(DefineHook fn, void* ctx) { defineHook_ = fn; defineCtx_ = ctx; }
+ /// The text the last sync wrote to its owner's status, so the owner can tell its own message from another's.
+ const char* statusText() const { return statusBuf_; }
+
// True means a new program was installed, which a modifier turns into a Layer rebuild.
/// Re-read the file and recompile when its content hash moved.
bool sync(const SysVarTable& sysvars, MoonModule& owner, const BuiltinTable& builtins) {
@@ -51,6 +54,8 @@ class MoonLiveScript {
return false;
resetPrintBudget();
+ // What the last program sized goes before this compile needs memory, which a small board does not have for both.
+ if (defineHook_) defineHook_(defineCtx_, DefinePhase::Compiling);
const char* err = nullptr;
uint32_t hash = 0;
if (compileScriptFile(engine_, name_, builtins, sysvars, err, &hash)) {
@@ -81,10 +86,10 @@ class MoonLiveScript {
if (n < 0) n = 0;
if (static_cast(n) > room) n = static_cast(room);
std::snprintf(statusBuf_ + n, sizeof(statusBuf_) - static_cast(n), "%s", at);
- owner.setStatus(statusBuf_, MoonModule::Severity::Error);
} else {
- owner.setStatus(err, MoonModule::Severity::Error);
+ std::snprintf(statusBuf_, sizeof(statusBuf_), "%s", err ? err : "compile failed");
}
+ owner.setStatus(statusBuf_, MoonModule::Severity::Error);
// Forget what the last script said: its strings are already overwritten in the pool.
dim_ = Dim::D2;
tags_ = nullptr;
diff --git a/src/core/moonlive/MoonLiveSpill.cpp b/src/core/moonlive/MoonLiveSpill.cpp
index b93b2e88e..a325fb686 100644
--- a/src/core/moonlive/MoonLiveSpill.cpp
+++ b/src/core/moonlive/MoonLiveSpill.cpp
@@ -125,22 +125,22 @@ bool spillToBudget(IrProgram& ir, const RegBudget& budget, uint8_t& slotsUsed) {
// --- 1. Find the loops, innermost first ----------------------------------------------------
- // Bounded by kIrLabels, since a loop needs a label.
- Loop loops[kIrLabels];
+ // Half of kIrLabels, since a loop takes two labels, its head and its exit.
+ Loop loops[kIrLabels / 2];
uint8_t loopCount = 0;
{
- int32_t labelAt[kIrLabels];
+ int16_t labelAt[kIrLabels]; // an op index, below kMaxIrOps
for (auto& p : labelAt) p = -1;
for (uint16_t i = 0; i < ir.count; i++)
if (ir.ops[i].op == IrOp::Label && ir.ops[i].imm >= 0 && ir.ops[i].imm < kIrLabels)
- labelAt[ir.ops[i].imm] = i;
+ labelAt[ir.ops[i].imm] = static_cast(i);
for (uint16_t i = 0; i < ir.count; i++) {
const IrInst& in = ir.ops[i];
if (in.op != IrOp::BranchNe) continue;
if (in.imm < 0 || in.imm >= kIrLabels) { spillDetail().guard = 5; return false; }; // an unbindable label: refuse
const int32_t tgt = labelAt[in.imm];
if (tgt < 0 || static_cast(tgt) > i) continue; // a forward branch, not a loop
- if (loopCount >= kIrLabels) { spillDetail().guard = 6; return false; };
+ if (loopCount >= kIrLabels / 2) { spillDetail().guard = 6; return false; };
loops[loopCount++] = {static_cast(tgt), i};
}
// Proper nesting is what makes innermost-first meaningful, so anything else is refused.
diff --git a/src/core/moonlive/moonlive_emit.h b/src/core/moonlive/moonlive_emit.h
index ed8aa38ab..ea8619bd3 100644
--- a/src/core/moonlive/moonlive_emit.h
+++ b/src/core/moonlive/moonlive_emit.h
@@ -7,7 +7,7 @@
/// @defgroup moonlive_emit MoonLive backend seam
/// @{
-/// A neutral declaration here, and the implementation per instruction set behind the platform boundary.
+/// A neutral declaration here, and the implementation per instruction set behind the platform boundary; the tests key on @xref{the-host-jit-macro|the host-JIT macro}.
///
/// @moreinfo
///
@@ -15,8 +15,7 @@
///
/// It is defined per platform, so the architecture check stays behind the boundary and this header stays neutral.
/// It exists for the test harness alone, and no shipping code branches on it: a build with no backend degrades at run time, reporting the failure and rendering dark.
-/// A test that calls render cannot be compiled there at all, which is why it is a macro rather than a constant.
-/// One answers the program, the other answers the build.
+/// A test that calls render cannot be compiled there at all, which is why it is a macro rather than a constant. One answers the program, the other answers the build.
namespace mm::moonlive {
@@ -41,11 +40,11 @@ size_t emitAnimatedFill(uint8_t* out, size_t cap);
struct IrProgram; // src/core/moonlive/MoonLiveIr.h
// Why the last lowering returned 0, since four distinct refusals once shared one return value.
-enum class LowerRefusal : uint8_t { None, Spill, NullCall, AsmOverflow, OverCap };
-inline LowerRefusal& lowerRefusal() { thread_local LowerRefusal r = LowerRefusal::None; return r; }
+enum class LowerRefusal : uint8_t { None, Spill, NullCall, AsmOverflow, OverCap, NoMemory };
+inline LowerRefusal& lowerRefusal() { MM_THREAD_LOCAL LowerRefusal r = LowerRefusal::None; return r; }
// Which allocator guard fired and the budget it saw, so a device names the cause.
struct SpillDetail { uint8_t guard = 0, avail = 0, temps = 0, vregs = 0, slots = 0, spilled = 0; };
-inline SpillDetail& spillDetail() { thread_local SpillDetail d; return d; }
+inline SpillDetail& spillDetail() { MM_THREAD_LOCAL SpillDetail d; return d; }
// What one target's register file offers the allocator, which is all the spill pass knows of an ISA.
struct RegBudget {
diff --git a/src/core/moonlive/moonlive_lower.h b/src/core/moonlive/moonlive_lower.h
index a4a18e110..eced95da2 100644
--- a/src/core/moonlive/moonlive_lower.h
+++ b/src/core/moonlive/moonlive_lower.h
@@ -5,6 +5,7 @@
#include "core/moonlive/MoonLiveSpill.h" // the register allocator, run before lowering
#include
+#include // placement new: the assembler lives on the heap
/// @defgroup moonlive_lower MoonLive lowering
/// @{
@@ -59,8 +60,17 @@ size_t lowerWith(IrProgram& ir, uint8_t* out, size_t cap, const RegBudget* squee
const RegId sAddr = static_cast(ir.vregsUsed); // per-channel address (both ops)
const RegId sCtr = static_cast(ir.vregsUsed + 1); // FillElems loop counter
+ // On the heap, since its label and fixup tables would put about 1 KB on the stack of every task that compiles.
+ struct HeapAsm {
+ A* p; ///< the assembler, built in place in its heap block
+ /// Destroy it and return its block, on every exit from the lowering.
+ ~HeapAsm() { if (p) { p->~A(); platform::free(p); } }
+ };
+ void* mem = platform::alloc(sizeof(A));
+ if (!mem) { lowerRefusal() = LowerRefusal::NoMemory; return 0; }
// Into the caller's buffer, so the assembler and the caller cannot disagree about capacity.
- A a(out, cap);
+ const HeapAsm owned{new (mem) A(out, cap)};
+ A& a = *owned.p;
using LabelId = decltype(a.newLabel());
// Derived from scratchTotal rather than fixed, so the reservation and the use cannot drift.
const RegId sHost = static_cast(ir.vregsUsed + scratchTotal - 1);
@@ -88,10 +98,13 @@ size_t lowerWith(IrProgram& ir, uint8_t* out, size_t cap, const RegBudget* squee
// Function number plus one while a guard is owed, since it is emitted after the parking.
int guardPending = 0;
+ // The all-ones index marks an unmade IR label, since an assembler hands out at most kAsmLabels below it.
+ static_assert(kAsmLabels <= LabelId(~LabelId(0)), "the all-ones label index must stay free as the unmade mark");
+ constexpr LabelId kUnmade = LabelId(~LabelId(0));
LabelId labels[kIrLabels];
- bool labelMade[kIrLabels] = {};
+ for (auto& l : labels) l = kUnmade;
auto labelFor = [&](int32_t id) -> LabelId {
- if (!labelMade[id]) { labels[id] = a.newLabel(); labelMade[id] = true; }
+ if (labels[id] == kUnmade) labels[id] = a.newLabel();
return labels[id];
};
diff --git a/src/core/moonlive/register-and-slot-contract.md b/src/core/moonlive/register-and-slot-contract.md
index bbc451772..ac0c4308d 100644
--- a/src/core/moonlive/register-and-slot-contract.md
+++ b/src/core/moonlive/register-and-slot-contract.md
@@ -1,9 +1,6 @@
# The MoonLive register and frame-slot contract
-One machine, three roles. This governs the compiler and the three backends, not a binding: a scripted
-layout, effect and modifier all compile through the same front end and the same lowering, so the
-ownership rules below hold identically whichever role a script plays. ("Layout" in this file means
-the memory layout of the register file and the frame — never the light-placement role.)
+One machine, three roles: a scripted layout, effect and modifier compile through the same front end and lowering. The ownership rules below so hold for the compiler and all four backends, whichever role a script plays. "Layout" here means the memory layout of the register file and the frame, never the light-placement role.
Who owns which register index, who owns which frame slot, and who is allowed to compute what from
`vregsUsed`. Written down because four places derive numbers from each other — core's compaction,
@@ -14,11 +11,11 @@ silently miscompiles on one target while the others stay green.
## Registers: one ascending layout, three owners
A vreg is an index. Each backend maps it to a machine register through its own table (`kXtReg`,
-`kRvReg`, `kArm64Reg`). The index space is carved as follows, **in this order, no gaps**:
+`kRvReg`, `kArm64Reg`, `kThReg`). The index space is carved as follows, **in this order, no gaps**:
| range | owner | meaning |
|---|---|---|
-| `0 .. kFirstTemp-1` | **core, fixed** | the host arguments: buf, nLights, cpl, t, ctrls. They arrive in registers at entry and every backend indexes them directly. |
+| `0 .. kFirstTemp-1` | **core, fixed** | the host arguments: buf, nLights, cpl, t, ctrls. They arrive in registers at entry, except the fifth on Thumb-2, which the prologue loads from the caller's stack; every backend then indexes them directly. |
| `kFirstTemp .. vregsUsed-1` | **core (parser, then spill pass)** | the program's values. The parser hands them out; the spill pass renumbers them down to what fits. |
| `vregsUsed .. vregsUsed+scratch-1` | **the backend** | the inline ops' scratch, computed per lowerer from `vregsUsed`. |
@@ -56,9 +53,10 @@ script that needs one spill.
## What `prologue(slots)` must reserve
-`slotsUsed` — the total from both owners, which `spillToBudget` returns. The backend turns a slot
-index into a frame offset its own way (Xtensa counts 4-byte words from `kFrameBase`; RISC-V and arm64
-build a frame in their prologue). Slot indices are the shared currency; offsets are private.
+`slotsUsed`: the total from both owners, which `spillToBudget` returns.
+The backend turns a slot index into a frame offset its own way.
+Xtensa counts 4-byte words from `kFrameBase`; RISC-V, arm64 and Thumb-2 build a frame in their prologue.
+Slot indices are the shared currency; offsets are private.
## Where a proposed change has to be checked
diff --git a/src/core/services/SdCardService.cpp b/src/core/services/SdCardService.cpp
index a351dce93..1c80f9a70 100644
--- a/src/core/services/SdCardService.cpp
+++ b/src/core/services/SdCardService.cpp
@@ -43,8 +43,8 @@ void SdCardService::defineControls() {
// Neither is hidden: there is no panel of this module's own, so the generic card is the only place either shows. Press-twice confirmed generically, the same contract every other confirm-gated button already has.
controls_.addButton("format", /*confirm=*/true);
- // Usage gauge, bound only once mounted: an absent/unreadable card has nothing to show; KiB is in the name since the shared Progress control's own label assumes bytes.
- if (mounted_ && totalKiB_ > 0) controls_.addProgress("card (KiB)", usedKiB_, totalKiB_, /*bytes=*/false);
+ // Usage gauge, bound only once mounted: an absent/unreadable card has nothing to show.
+ if (mounted_ && totalKiB_ > 0) controls_.addProgress("card", usedKiB_, totalKiB_, ProgressUnit::KiB);
MoonModule::defineControls();
}
diff --git a/src/core/services/SdCardService.h b/src/core/services/SdCardService.h
index 46c323c98..a7727ce33 100644
--- a/src/core/services/SdCardService.h
+++ b/src/core/services/SdCardService.h
@@ -38,9 +38,9 @@ namespace mm {
///
/// The card mounts at `/sd`, so the existing File Manager tree already shows it; this module owns only pins, the gauge and format.
///
-/// ## Why KiB, not the shared Progress control's bytes
+/// ## Why the gauge counts KiB
///
-/// A card's byte count overruns the `uint32_t` the shared Progress control carries, so `platform::sdCardUsed`/`Total` are `uint64_t` and this module narrows to KiB before handing the gauge its numbers.
+/// A card's byte count overruns the `uint32_t` a Progress control carries, so `platform::sdCardUsed`/`Total` are `uint64_t` and this module narrows to KiB, which the gauge declares as its unit.
///
/// Prior art: I2cBusModule's live pin reconfiguration; FileManagerModule's usage gauge.
class SdCardService : public MoonModule {
@@ -61,13 +61,16 @@ class SdCardService : public MoonModule {
void tick1s() MM_NONBLOCKING override;
/// The chip a board preset is wired for; Custom is Any, which fits every chip.
- enum class SdPresetChip : uint8_t { Any, Classic, P4 };
+ enum class SdPresetChip : uint8_t { Any, Classic, P4, Teensy };
/// What a build can wire, so the filter answers for any chip; a classic is the chip that is none of the others.
- struct SdChip { bool previews, classic, p4; };
- static constexpr SdChip kThisChip{platform::previewsSdPresets, !platform::isEsp32P4 && !platform::isEsp32S3 && !platform::isEsp32S31, platform::isEsp32P4};
+ struct SdChip { bool previews, classic, p4, teensy; };
+ static constexpr SdChip kThisChip{platform::previewsSdPresets,
+ !platform::isEsp32P4 && !platform::isEsp32S3 && !platform::isEsp32S31 && !platform::isTeensy41,
+ platform::isEsp32P4, platform::isTeensy41};
/// Whether a preset for `chip` belongs on `on`: Custom everywhere, and every preset where the platform previews them.
static constexpr bool presetFits(SdPresetChip chip, SdChip on = kThisChip) {
- return chip == SdPresetChip::Any || on.previews || (chip == SdPresetChip::Classic && on.classic) || (chip == SdPresetChip::P4 && on.p4);
+ return chip == SdPresetChip::Any || on.previews || (chip == SdPresetChip::Classic && on.classic) ||
+ (chip == SdPresetChip::P4 && on.p4) || (chip == SdPresetChip::Teensy && on.teensy);
}
/// Whether `on` opens on its one fitting preset rather than Custom: @xref{a-board-preset-names-a-soldered-wiring-as-ethernets-does}.
static constexpr bool opensOnSolePreset(SdChip on = kThisChip) {
@@ -81,7 +84,7 @@ class SdCardService : public MoonModule {
bool formatting_ = false; ///< a format runs in the background, which tick1s() waits out
bool tried_ = false; ///< whether prepare() has tried triedPins_ since the last release()
platform::SdPinConfig triedPins_{}; ///< the pins prepare() last tried, mounted or not, so an unrelated prepare() is a no-op
- uint32_t usedKiB_ = 0; ///< card usage, in KiB: @xref{why-kib-not-the-shared-progress-controls-bytes}
+ uint32_t usedKiB_ = 0; ///< card usage, in KiB: @xref{why-the-gauge-counts-kib}
uint32_t totalKiB_ = 0; ///< the card's capacity, in KiB, read once on mount
uint8_t secondsSinceScan_ = 0; ///< paces the usage scan
@@ -111,6 +114,8 @@ class SdCardService : public MoonModule {
{"Classic SDMMC (1-bit)", SdPresetChip::Classic, platform::SdBus::Sdmmc, 14, 15, 2, -1, -1, -1, -1, false},
// SPI mode as the QuinLED Dig-Octa wires it: SCK 14, MOSI 15, MISO 36, CS 16.
{"Classic SPI", SdPresetChip::Classic, platform::SdBus::Spi, 14, 15, 36, -1, -1, 16, -1, false},
+ // The Teensy 4.1's built-in socket on its SDIO pins, which no other wiring reaches; the platform's isSocketWiring refuses any other set, so the two change together.
+ {"Teensy 4.1 SD socket", SdPresetChip::Teensy, platform::SdBus::Sdmmc, 44, 45, 43, 42, 47, 46, -1, false},
{"Custom", SdPresetChip::Any, platform::SdBus::Sdmmc, -1, -1, -1, -1, -1, -1, -1, true},
};
static constexpr uint8_t kSdPresetCount = sizeof(kSdPresets) / sizeof(kSdPresets[0]);
diff --git a/src/core/system/EthernetModule.h b/src/core/system/EthernetModule.h
index d1229fffe..8afe944bc 100644
--- a/src/core/system/EthernetModule.h
+++ b/src/core/system/EthernetModule.h
@@ -62,8 +62,9 @@ class EthernetModule : public MoonModule {
controls_.addSelect("ethBoard", ethPresetSel_, ethPresetOptions_, ethPresetCount_);
// By label: the list is filtered per build, so an index would name a different board.
controls_.setPersistLabel(controls_.count() - 1);
- // Hidden on a known board, and hidden stays BOUND: the values still drive the interface.
- const bool editable = ethPinsEditable();
+ controls_.setHidden(controls_.count() - 1, platform::ethPhyIsFixed);
+ // Hidden on a known board or a fixed interface, and hidden stays BOUND: the values still drive the interface.
+ const bool editable = !platform::ethPhyIsFixed && ethPinsEditable();
controls_.addSelect("ethType", ethType_, ethTypeOptions_, 5);
controls_.setHidden(controls_.count() - 1, !editable);
const bool isRmii = (ethType_ == platform::ethLan8720 || ethType_ == platform::ethIp101);
@@ -129,9 +130,7 @@ class EthernetModule : public MoonModule {
void syncConfig() {
if constexpr (platform::hasEthernet) {
platform::EthPinConfig cfg{};
- // Where the platform fixes the interface, its own type wins.
- cfg.phyType = platform::ethPhyIsFixed
- ? static_cast(platform::ethConfigDefault.phyType) : ethType_;
+ cfg.phyType = phyType();
cfg.phyAddr = ethPhyAddr_;
cfg.mdcGpio = ethMdcGpio_;
cfg.mdioGpio = ethMdioGpio_;
@@ -206,20 +205,21 @@ class EthernetModule : public MoonModule {
/// Whether a wired interface is set up, which is what lets the access point stay closed.
bool configured() const MM_NONBLOCKING {
if constexpr (!platform::hasEthernet) return false;
- const uint8_t type = platform::ethPhyIsFixed ? static_cast(platform::ethConfigDefault.phyType) : ethType_;
- return type != static_cast(platform::ethNone);
+ return phyType() != static_cast(platform::ethNone);
}
/// Pin the configured address onto the wired interface, or do nothing where a client runs.
void applyStatic() const { ip_.applyStatic(platform::NetIface::Eth); }
/// What a chip can wire, so the board filter answers for any chip rather than only the one built for.
- struct EthChip { bool drivesEthernet, w5500, p4, s31; };
- static constexpr EthChip kThisChip{platform::hasEthernet, platform::hasEthW5500, platform::isEsp32P4, platform::isEsp32S31};
+ struct EthChip { bool drivesEthernet, w5500, p4, s31, fixed; };
+ static constexpr EthChip kThisChip{platform::hasEthernet, platform::hasEthW5500, platform::isEsp32P4, platform::isEsp32S31,
+ platform::ethPhyIsFixed};
// A preset carries a pin map as well as a PHY, so one the chip cannot wire persists pins reaching nothing.
/// Whether a board preset with PHY `type` belongs on `chip`: Custom everywhere, and every board on the desktop, which drives none and previews them all.
static constexpr bool presetFits(int8_t type, EthChip chip = kThisChip) {
if (type == 0 || !chip.drivesEthernet) return true;
+ if (chip.fixed) return false; // the build chose the interface, so no board wiring applies
if (type == 3) return chip.w5500; // SPI, a separate driver
if (chip.w5500) return false;
if (type == 2) return chip.p4; // IP101: the P4's
@@ -335,6 +335,11 @@ class EthernetModule : public MoonModule {
return kEthPresets[ethPresetIndex_[ethPresetSel_]].editable;
}
+ /// The interface this device drives: where the platform fixes it, the platform's own type wins over the control.
+ uint8_t phyType() const MM_NONBLOCKING {
+ return platform::ethPhyIsFixed ? static_cast(platform::ethConfigDefault.phyType) : ethType_;
+ }
+
/// Are the pin controls the user's to edit, for the preset currently selected?
bool ethPinsEditable() const {
if (ethPresetSel_ >= ethPresetCount_) return true; // nothing resolved: never hide
diff --git a/src/core/system/FileManagerModule.cpp b/src/core/system/FileManagerModule.cpp
index a47ff2c3e..de5c0c5f2 100644
--- a/src/core/system/FileManagerModule.cpp
+++ b/src/core/system/FileManagerModule.cpp
@@ -10,29 +10,19 @@
namespace mm {
void FileManagerModule::defineControls() {
- // Only `show hidden` is a control: the whole File Manager surface is the tree panel (app.js renderFileManager), which lists over /api/dir and reads the gauges from /api/state.
- // The mkdir/delete OPS are their own HTTP endpoints (POST/DELETE /api/dir?path=), not persisted controls, so a create/delete carries its path in the request, not in device storage.
- // The `show hidden` flag keeps it bound for the API while the generic control list skips it.
+ // `show hidden` is the one setting: the File Manager's surface is the tree panel, and the flag stays bound for the API while the generic control list skips it.
controls_.addControl("show hidden", showHidden_); // reveal dot-prefixed entries (e.g. .config)
controls_.setHidden(controls_.count() - 1, true);
- // Filesystem-usage gauge (used / total bytes), read from the platform.
- // Shown below the tree in the panel, the File Manager is where filesystem space is relevant, so it owns the control.
- // Bound only when the platform reports a real partition (desktop / a no-data-partition chip reports 0).
- // Read the total ONCE (it's fixed) the first time controls are built; the used value is refreshed only on the throttled tick1s. defineControls() is re-runnable (a Select can rebuild the control set on any control write), so it must NOT re-scan the filesystem here, a LittleFS usage scan walks blocks and isn't free.
- // Rebuild from the cached values instead.
+ // The `flash` gauge, bound when the platform reports a partition: the total is read once and the usage on tick1s, since defineControls re-runs and a LittleFS usage scan walks every block.
if (totalBytes_ == 0) {
totalBytes_ = static_cast(platform::filesystemTotal());
usedBytes_ = static_cast(platform::filesystemUsed());
}
if (totalBytes_ > 0) {
- controls_.addProgress("filesystem", usedBytes_, totalBytes_);
+ controls_.addProgress("flash", usedBytes_, totalBytes_);
controls_.setHidden(controls_.count() - 1, true); // renders as the usage bar in the panel, not generically
}
- // "last saved" readout, how long ago config was persisted.
- // The value is OWNED by the FilesystemModule engine (non-UI); the File Manager just displays it here (this is where filesystem state is topical).
- // Bind the control straight to the engine's live buffer, no per-instance copy, the same no-copy pattern SystemModule uses for its static strings.
- // The engine is the boot-wired singleton (alive for the device's life), and its tick1s keeps the string current.
- // Bound only when the engine exists (it's constructed before this module).
+ // How long ago config was saved, bound straight to the FilesystemModule engine's buffer, which its tick1s keeps current, as SystemModule binds its strings.
if (FilesystemModule* fs = FilesystemModule::instance()) {
controls_.addReadOnly("lastSaved", fs->lastSavedStr());
controls_.setHidden(controls_.count() - 1, true); // shown in the panel header, not generically
@@ -41,13 +31,7 @@ void FileManagerModule::defineControls() {
}
void FileManagerModule::tick1s() MM_NONBLOCKING {
- // ONCE A MINUTE, not once a second.
- // `filesystemUsed()` is `esp_littlefs_info`, which walks every block of the partition to count what is in use.
- // Measured at ~80 ms on an S3, and tick1s runs INLINE on the render thread, so at 1 Hz it stuttered the fixture once a second.
- // A particle effect made it obvious where a shader had hidden it: a shader redraws each frame from the clock and simply misses one, while a particle integrates the stall into its trajectory and visibly jumps (FrameTime spends the whole gap, by design).
- //
- // The value feeds one progress bar on this card, so a minute-old figure is no worse to a reader and the scan stops being a per-second cost.
- // Anything needing an exact figure should read it directly rather than this cache.
+ // Once a minute: `filesystemUsed()` walks every block, about 80 ms on an S3 inline on the render thread, and a minute-old figure serves this one gauge as well.
if (totalBytes_ == 0) return;
if (++secondsSinceScan_ < 60) return;
secondsSinceScan_ = 0;
@@ -56,13 +40,11 @@ void FileManagerModule::tick1s() MM_NONBLOCKING {
void FileManagerModule::setup() {
MoonModule::setup();
- // `show hidden` is a transient view preference, not device config, force it off on every boot regardless of any persisted value (setup() runs after persistence overlays it). A file manager opens with hidden entries hidden; the user re-toggles per session.
+ // `show hidden` is a view preference, off on every boot whatever was saved: setup() runs after persistence overlays it.
showHidden_ = false;
}
-// mkdir/delete are HTTP endpoints (POST/DELETE /api/dir?path=) in HttpServerModule.
-// A create/delete carries its path in the request and touches the filesystem directly, so this module holds no op state and writes nothing to persisted config.
-// The path guard (reject `..`, root at mount) lives once in HttpServerModule::parseFilePath, shared with /api/file + /api/dir GET.
+// Creating and deleting are HttpServerModule's /api/dir routes, which carry the path in the request, so this module holds no op state; parseFilePath is the one path guard.
} // namespace mm
diff --git a/src/core/system/HttpServerModule.cpp b/src/core/system/HttpServerModule.cpp
index 7fb21458d..4965afc4b 100644
--- a/src/core/system/HttpServerModule.cpp
+++ b/src/core/system/HttpServerModule.cpp
@@ -61,6 +61,7 @@
#include "platform/platform.h"
#include "ui/ui_embedded.h"
+#include // std::min: a stream's span within its budget
#include
#include
#include
@@ -106,13 +107,9 @@ void HttpServerModule::onValuesChanged(MoonModule* mod) {
}
void HttpServerModule::release() {
- // Drop the in-flight sends before the clients go: the preview frame borrows its buffer, the state frame owns its JSON body.
+ // Drop the in-flight sends before the clients go: the preview frame borrows its buffer, a state stream owns its fragment.
cancelBufferedSend();
- if (stateSend_.active) {
- platform::free(const_cast(stateSend_.body));
- stateSend_.body = nullptr;
- stateSend_.active = false;
- }
+ for (auto& st : streams_) endStream(st);
for (auto& ws : wsClients_) ws.close();
for (auto& c : parked_) c.close();
releaseSdTransfers();
@@ -135,15 +132,15 @@ void HttpServerModule::tick20ms() MM_NONBLOCKING {
// Drain the in-flight preview frame on the 20 ms transport cadence, before accept so a connection burst cannot starve it; sockets stay off the render tick (architecture.md § Parallelism).
drainPreviewSend();
drainStateSend();
+ // Inbound WS frames before the push, so the subscription a page sends on open shapes its first document: control writes, subscriptions, and the WLED app's {on,bri}.
+ pollUplinks();
// A pending full resync is served here, not on the 1 s tick, so a connect reaches its first preview within tens of ms; the flag gates the expensive serialize.
- if (fullResyncPending_ || soonCount_) {
- const SecretsHidden hide(anyWsClientOnAccessPoint()); // one push serves every client
- if (fullResyncPending_) pushStateToWebSockets();
+ if (fullResyncPending_ || anyClientResync() || soonCount_) {
+ const SecretsHidden hide(anyWsClientOnAccessPoint()); // one patch serves every client; a stream hides per client
+ if (fullResyncPending_ || anyClientResync()) pushStateToWebSockets();
// A module that asked goes out now rather than on the next second.
if (soonCount_) pushSoonPatch();
}
- // Inbound WS frames: the WLED app sets on/off and brightness with an {on,bri} text frame over /ws.
- pollWledStateFromWebSockets();
serveConnections();
if constexpr (platform::hasSdCard) driveSdTransfers(); // sdTransfers_ has zero slots otherwise, so there is nothing to drive
}
@@ -1230,42 +1227,82 @@ void HttpServerModule::serveState(platform::TcpConnection& conn) {
}
void HttpServerModule::buildStateJson(JsonSink& sink) {
- sink.append("{\"modules\":[");
+ TreeCursor c;
+ writeTreeFragment(sink, c, 0);
+}
- if (scheduler_) {
- bool first = true;
- for (uint8_t m = 0; m < scheduler_->moduleCount(); m++) {
- auto* mod = scheduler_->module(m);
- // Skip modules whose appearsInUi() is false (HttpServerModule, FilesystemModule): the one mechanism for "not a card in /api/state".
- if (!mod || !mod->appearsInUi()) continue;
- if (!first) sink.append(",");
- first = false;
- writeModuleJson(sink, mod);
- }
+// The diff-on-the-wire cache keeps an 8-byte FNV-1a {path,value} hash per leaf, not the value; a collision skips one update at worst.
+
+bool HttpServerModule::Subscription::names_(const char* s) const {
+ if (!s || !s[0]) return false;
+ const uint32_t h = fnv1a(s, std::strlen(s));
+ for (uint8_t i = 0; i < count; i++)
+ if (names[i] == h) return true;
+ return false;
+}
+
+bool HttpServerModule::Subscription::covers(const MoonModule* mod) const {
+ if (all) return true;
+ for (const MoonModule* m = mod; m && count; m = m->parent())
+ if (names_(m->name()) || names_(m->typeName())) return true;
+ return false;
+}
+
+bool HttpServerModule::Subscription::parse(const char* body) {
+ json::JsonDoc* doc = nullptr;
+ const json::JsonNode* arr = json::parseListArray(body, "subscribe", doc);
+ if (!arr) return false;
+ *this = Subscription{};
+ received = true;
+ const int n = json::arraySize(*doc, arr);
+ for (int i = 0; i < n; i++) {
+ const json::JsonNode* el = json::element(*doc, arr, i);
+ if (!el || el->type != json::JsonType::String || !el->str || !el->str[0]) continue;
+ if (std::strcmp(el->str, "*") == 0) all = true;
+ else if (count < kMaxModules) names[count++] = fnv1a(el->str, std::strlen(el->str));
}
+ return true;
+}
- sink.append("]}");
+bool HttpServerModule::Subscription::operator==(const Subscription& o) const {
+ if (received != o.received || all != o.all || count != o.count) return false;
+ for (uint8_t i = 0; i < count; i++)
+ if (names[i] != o.names[i]) return false;
+ return true;
}
-// The diff-on-the-wire cache keeps an 8-byte FNV-1a {path,value} hash per leaf, not the value; a collision skips one update at worst.
+bool HttpServerModule::anyClientCovers(const MoonModule* mod) const {
+ // Every slot's subscription, live or not: the uplink poll clears a slot's subscription once its client is gone.
+ for (const auto& sub : subs_)
+ if (sub.covers(mod)) return true;
+ return false;
+}
-// The diff-on-the-wire core: visit every UI leaf the periodic push sends, in buildStateJson order, as fn(pathHash, valueHash, path, valueSink); names are unique tree-wide, so "/" is stable.
+// The diff-on-the-wire core: visit every UI leaf a client subscribes to, in buildStateJson order, as fn(module, pathHash, valueHash, path, valueSink); names are unique tree-wide, so "/" is stable.
template
-void HttpServerModule::forEachStateLeaf(Fn&& fn) {
+void HttpServerModule::forEachStateLeaf(Fn&& fn, bool everything) {
if (!scheduler_) return;
// `fn`, not std::forward: forwarding inside a loop moves the callable on the first module, leaving later modules a moved-from object.
for (uint8_t m = 0; m < scheduler_->moduleCount(); m++)
if (auto* mod = scheduler_->module(m))
- if (mod->appearsInUi()) visitModuleLeaves(mod, fn);
+ if (mod->appearsInUi()) visitModuleLeaves(mod, fn, everything);
}
template
-void HttpServerModule::visitModuleLeaves(MoonModule* mod, Fn&& fn) {
+void HttpServerModule::visitModuleLeaves(MoonModule* mod, Fn&& fn, bool everything) {
+ if (everything || anyClientCovers(mod)) visitOwnLeaves(mod, fn);
+ // A module nobody views can still hold one somebody does, the preview inside Drivers.
+ for (uint8_t i = 0; i < mod->childCount(); i++)
+ if (auto* ch = mod->child(i)) visitModuleLeaves(ch, fn, everything);
+}
+
+template
+void HttpServerModule::visitOwnLeaves(MoonModule* mod, Fn&& fn) {
char path[80];
// Header telemetry the UI shows per card, `@`-prefixed so it cannot collide with a control name; only fields that change per tick (timing, memory) ride the patch.
auto leaf = [&](const char* fieldPath, const char* valueJson) {
JsonSink vs; vs.append(valueJson);
- fn(fnv1a(fieldPath, std::strlen(fieldPath)), fnv1a(vs.data(), vs.size()), fieldPath, vs);
+ fn(mod, fnv1a(fieldPath, std::strlen(fieldPath)), fnv1a(vs.data(), vs.size()), fieldPath, vs);
};
char num[24];
std::snprintf(path, sizeof(path), "%s/@tickTimeUs", mod->name());
@@ -1295,11 +1332,8 @@ void HttpServerModule::visitModuleLeaves(MoonModule* mod, Fn&& fn) {
JsonSink vs;
if (c.type == ControlType::Password) writeObfuscatedPassword(vs, static_cast(c.ptr)); // as the full state shows it
else writeControlValue(vs, c);
- fn(fnv1a(path, std::strlen(path)), fnv1a(vs.data(), vs.size()), path, vs);
+ fn(mod, fnv1a(path, std::strlen(path)), fnv1a(vs.data(), vs.size()), path, vs);
}
- // `fn`, not std::forward, for the same reason as the caller: forwarding in a loop moves the callable into the first child.
- for (uint8_t i = 0; i < mod->childCount(); i++)
- if (auto* ch = mod->child(i)) visitModuleLeaves(ch, fn);
}
// Look up a leaf's cached value-hash by path-hash, or nullptr; a linear scan over the flat cache is a handful of int compares for ~92 leaves.
@@ -1309,71 +1343,118 @@ HttpServerModule::LeafHash* HttpServerModule::findLeaf(uint32_t pathHash) {
return nullptr;
}
-void HttpServerModule::baselineLeafHashes() {
- // Count the leaves, size the buffer exactly, then fill from scratch (resize does not preserve); off the hot path, on a full-state resync only.
- uint16_t n = 0;
- forEachStateLeaf([&](uint32_t, uint32_t, const char*, JsonSink&) { n++; });
+void HttpServerModule::baselineLeafHashes(bool everything) {
+ // Count the leaves and the modules, size the hashes and the patch slices exactly, then fill from scratch, since resize does not preserve; this runs on a resync, not each second.
+ uint16_t n = 0, modules = 0;
+ const MoonModule* last = nullptr;
+ forEachStateLeaf([&](const MoonModule* mod, uint32_t, uint32_t, const char*, JsonSink&) {
+ n++;
+ if (mod != last) { modules++; last = mod; }
+ }, everything);
leafHashes_.resize(n);
leafHashCount_ = 0;
- forEachStateLeaf([&](uint32_t ph, uint32_t vh, const char*, JsonSink&) {
+ forEachStateLeaf([&](const MoonModule*, uint32_t ph, uint32_t vh, const char*, JsonSink&) {
if (leafHashCount_ < leafHashes_.count()) leafHashes_[leafHashCount_++] = {ph, vh};
- });
+ }, everything);
+ if (patchSlices_.count() < modules) patchSlices_.resize(modules);
}
uint16_t HttpServerModule::buildStatePatch(JsonSink& sink) {
- return buildPatch(sink, [&](auto&& fn) { forEachStateLeaf(fn); });
+ JsonSink entries;
+ uint16_t slices = 0;
+ collectPatch(entries, slices, [&](auto&& fn) { forEachStateLeaf(fn, true); });
+ return writeClientPatch(sink, entries, slices, nullptr);
}
-uint16_t HttpServerModule::buildSoonPatch(JsonSink& sink) {
- const uint16_t changed = buildPatch(sink, [&](auto&& fn) {
- for (uint8_t i = 0; i < soonCount_; i++)
- // No appearsInUi test: the interface drops a patch for a module it never saw.
- if (MoonModule* mod = scheduler_ ? scheduler_->firstByName(soon_[i]) : nullptr) visitModuleLeaves(mod, fn);
- });
+void HttpServerModule::buildClientPatchesForTest(JsonSink* outs, int count) {
+ JsonSink entries;
+ uint16_t slices = 0;
+ collectPatch(entries, slices, [&](auto&& fn) { forEachStateLeaf(fn); });
+ for (int i = 0; i < count && i < MAX_WS_CLIENTS; i++) writeClientPatch(outs[i], entries, slices, &subs_[i]);
+}
+
+template
+void HttpServerModule::forEachSoonLeaf(Fn&& fn, bool everything) {
+ for (uint8_t i = 0; i < soonCount_; i++)
+ // No appearsInUi test: the interface drops a patch for a module it never saw.
+ if (MoonModule* mod = scheduler_ ? scheduler_->firstByName(soon_[i]) : nullptr) visitModuleLeaves(mod, fn, everything);
soonCount_ = 0;
- return changed;
+}
+
+uint16_t HttpServerModule::buildSoonPatch(JsonSink& sink) {
+ JsonSink entries;
+ uint16_t slices = 0;
+ collectPatch(entries, slices, [&](auto&& fn) { forEachSoonLeaf(fn, true); });
+ return writeClientPatch(sink, entries, slices, nullptr);
}
void HttpServerModule::pushSoonPatch() {
- // A small frame written into a full state mid-drain would land inside that message, so it waits.
- if (fullResyncPending_ || stateSend_.active) return;
+ // A small frame written into a full state mid-stream would land inside that message, so it waits.
+ if (fullResyncPending_ || anyClientResync() || anyStreamActive()) return;
const uint32_t now = platform::millis();
if (now - soonSentMs_ < kSoonMs) return;
soonSentMs_ = now;
- JsonSink sink;
// The leaves' hashes update as they go, so the second's patch does not send them again.
- if (buildSoonPatch(sink) > 0) sendPatch(sink);
-}
-
-void HttpServerModule::sendPatch(const JsonSink& sink) {
- for (auto& ws : wsClients_) {
- if (!ws.valid()) continue;
- if (!sendWsTextFrame(ws, sink.data(), static_cast(sink.size()))) ws.close();
- }
+ sendPatches([&](auto&& fn) { forEachSoonLeaf(fn); });
}
template
-uint16_t HttpServerModule::buildPatch(JsonSink& sink, Walk&& walk) {
- sink.append("{\"patch\":[");
+uint16_t HttpServerModule::collectPatch(JsonSink& entries, uint16_t& sliceCount, Walk&& walk) {
+ sliceCount = 0;
uint16_t changed = 0;
- walk([&](uint32_t ph, uint32_t vh, const char* path, JsonSink& vs) {
+ walk([&](const MoonModule* mod, uint32_t ph, uint32_t vh, const char* path, JsonSink& vs) {
LeafHash* h = findLeaf(ph);
if (h && h->value == vh) return; // unchanged: the common case, emit nothing
+ const bool opens = !sliceCount || patchSlices_[sliceCount - 1].mod != mod;
+ // No room for the module's slice leaves the hash as it was, so the change goes out once there is.
+ if (opens && sliceCount >= patchSlices_.count()) return;
if (h) h->value = vh; // known leaf, value changed → update cache
// A leaf missing from the baseline cannot happen, since every structural mutation re-baselines first; emit it without growing the cache, because ScratchBuffer::resize is non-preserving and would discard every hash.
- if (changed++) sink.append(",");
- sink.append("{\"path\":\"");
- sink.append(path);
- sink.append("\",\"value\":");
- sink.append(vs.data()); // the already-serialized value
- sink.append("}");
+ if (opens) patchSlices_[sliceCount++] = {mod, static_cast(entries.size()), 0, 0};
+ entries.append(",{\"path\":\"");
+ entries.append(path);
+ entries.append("\",\"value\":");
+ entries.append(vs.data(), vs.size()); // the already-serialized value
+ entries.append("}");
+ PatchSlice& s = patchSlices_[sliceCount - 1];
+ s.end = static_cast(entries.size());
+ s.count++;
+ changed++;
});
- sink.append("]}");
return changed;
}
-void HttpServerModule::writeModuleJson(JsonSink& sink, MoonModule* mod) {
- // Per-module header: name, role, enabled, tickTimeUs (fps/ms display), classSize (static C++ object bytes) + dynamicBytes (heap), controls
+uint16_t HttpServerModule::writeClientPatch(JsonSink& out, const JsonSink& entries, uint16_t sliceCount, const Subscription* sub) {
+ out.append("{\"patch\":[");
+ uint16_t n = 0;
+ for (uint16_t i = 0; i < sliceCount; i++) {
+ const PatchSlice& s = patchSlices_[i];
+ if (sub && !sub->covers(s.mod)) continue;
+ // Every entry leads with its comma, which the first one of the frame drops.
+ const size_t skip = n == 0 ? 1 : 0;
+ out.append(entries.data() + s.begin + skip, s.end - s.begin - skip);
+ n += s.count;
+ }
+ out.append("]}");
+ return n;
+}
+
+template
+void HttpServerModule::sendPatches(Walk&& walk, bool skipResyncing) {
+ JsonSink entries;
+ uint16_t slices = 0;
+ if (collectPatch(entries, slices, walk) == 0) return;
+ for (int i = 0; i < MAX_WS_CLIENTS; i++) {
+ auto& ws = wsClients_[i];
+ if (!ws.valid() || (skipResyncing && resyncClient_[i])) continue;
+ JsonSink out;
+ if (writeClientPatch(out, entries, slices, &subs_[i]) == 0) continue;
+ if (!sendWsTextFrame(ws, out.data(), static_cast(out.size()))) ws.close();
+ }
+}
+
+void HttpServerModule::writeModuleHeader(JsonSink& sink, MoonModule* mod) {
+ // Per-module header: name, role, enabled, tickTimeUs (fps/ms display), classSize (static C++ object bytes) + dynamicBytes (heap)
const char* roleStr = roleName(mod->role());
const char* type = mod->typeName();
if (!type) type = "";
@@ -1400,22 +1481,93 @@ void HttpServerModule::writeModuleJson(JsonSink& sink, MoonModule* mod) {
writeStatus(sink, mod);
// userEditable is omitted when true (the UI treats absent as editable); modules that opt out, like PreviewDriver, hide their delete/replace affordance.
if (!mod->userEditable()) sink.append(",\"userEditable\":false");
- sink.append(",\"controls\":[");
- writeControls(sink, mod);
- sink.append("]");
+}
- // Children
- uint8_t cc = mod->childCount();
- if (cc > 0) {
- sink.append(",\"children\":[");
- for (uint8_t i = 0; i < cc; i++) {
- if (i > 0) sink.append(",");
- writeModuleJson(sink, mod->child(i));
- }
+void HttpServerModule::writeModuleJson(JsonSink& sink, MoonModule* mod) {
+ TreeCursor c;
+ c.single = mod;
+ writeTreeFragment(sink, c, 0);
+}
+
+MoonModule* HttpServerModule::moduleAt(const TreeCursor& c) const {
+ MoonModule* mod = c.single ? (c.idx[0] == 0 ? c.single : nullptr)
+ : (scheduler_ ? scheduler_->module(c.idx[0]) : nullptr);
+ for (uint8_t l = 1; mod && l < c.depth; l++)
+ mod = c.idx[l] < mod->childCount() ? mod->child(c.idx[l]) : nullptr;
+ return mod;
+}
+
+bool HttpServerModule::writeTreeFragment(JsonSink& sink, TreeCursor& c, size_t target) {
+ if (c.done) return true;
+ if (c.depth == 0) {
+ if (!c.single) sink.append("{\"modules\":[");
+ c.depth = 1;
+ c.idx[0] = 0;
+ c.phase = TreeCursor::kHeader;
+ }
+ // The path is resolved again at every step rather than held as pointers, so a tree changed between fragments still ends in valid JSON.
+ while (target == 0 || sink.size() < target) {
+ if (MoonModule* mod = moduleAt(c)) stepInto(sink, c, mod);
+ else if (stepPast(sink, c)) return true;
+ }
+ return false;
+}
+
+bool HttpServerModule::stepPast(JsonSink& sink, TreeCursor& c) {
+ const uint8_t level = c.depth - 1;
+ if (c.phase != TreeCursor::kHeader) {
+ // The module this level was writing went mid-object: close what it opened, as if it ended there, and go on with its next sibling.
+ sink.append(c.phase == TreeCursor::kControls ? "]}" : "}");
+ c.idx[level]++;
+ c.phase = TreeCursor::kHeader;
+ return false;
+ }
+ if (level > 0) {
+ sink.append("]}"); // the parent's children and the parent itself
+ c.depth--;
+ c.idx[c.depth - 1]++;
+ return false;
+ }
+ if (!c.single) {
sink.append("]");
+ // The id of the request the streamed state answers, so the page knows its view arrived rather than an earlier one.
+ if (c.answers) sink.appendf(",\"resync\":%lu", static_cast(c.answers));
+ sink.append("}");
}
+ c.done = true;
+ return true;
+}
- sink.append("}");
+void HttpServerModule::stepInto(JsonSink& sink, TreeCursor& c, MoonModule* mod) {
+ const uint8_t level = c.depth - 1;
+ if (c.phase == TreeCursor::kHeader) {
+ // HttpServerModule and FilesystemModule are no cards: appearsInUi is the one mechanism for that.
+ if (level == 0 && !c.single && !mod->appearsInUi()) { c.idx[0]++; return; }
+ if (level == 0 ? c.wroteRoot : c.idx[level] > 0) sink.append(",");
+ if (level == 0) c.wroteRoot = true;
+ writeModuleHeader(sink, mod);
+ // An outline leaves the controls out: the module's name and shape are what a client viewing elsewhere reads.
+ const bool full = !c.sub || c.sub->covers(mod);
+ if (full) sink.append(",\"controls\":[");
+ c.ctrl = 0;
+ c.phase = full ? TreeCursor::kControls : TreeCursor::kChildren;
+ } else if (c.phase == TreeCursor::kControls) {
+ if (c.ctrl < mod->controls().count()) {
+ if (c.ctrl > 0) sink.append(",");
+ writeControl(sink, mod->controls()[c.ctrl++]);
+ return;
+ }
+ sink.append("]");
+ c.phase = TreeCursor::kChildren;
+ } else if (mod->childCount() > 0 && c.depth < TreeCursor::kMaxDepth) {
+ sink.append(",\"children\":[");
+ c.idx[c.depth++] = 0;
+ c.phase = TreeCursor::kHeader;
+ } else {
+ sink.append("}");
+ c.idx[level]++;
+ c.phase = TreeCursor::kHeader;
+ }
}
void HttpServerModule::writeStatus(JsonSink& sink, MoonModule* mod) {
@@ -1429,48 +1581,43 @@ void HttpServerModule::writeStatus(JsonSink& sink, MoonModule* mod) {
sink.appendf(",\"severity\":\"%s\"", sevStr[static_cast(mod->severity())]);
}
-void HttpServerModule::writeControls(JsonSink& sink, MoonModule* mod) {
- auto& ctrls = mod->controls();
- for (uint8_t i = 0; i < ctrls.count(); i++) {
- if (i > 0) sink.append(",");
- auto& c = ctrls[i];
- // Common wrapper for every control; per-type VALUE + EXTRAS live in Control.cpp, but Password is handled here since its API form is XOR-obfuscated + base64 and writeControlValue stays sink-neutral.
- sink.appendf("{\"name\":\"%s\",\"type\":\"%s\",\"value\":",
- c.name, controlTypeName(c.type));
- if (c.type == ControlType::Password) {
- // Not readable at a glance in `curl /api/state`, and the UI's hold-to-peek reverses it.
- writeObfuscatedPassword(sink, static_cast(c.ptr));
- } else {
- writeControlValue(sink, c);
- }
- writeControlMetadata(sink, c);
- // Emit optional flags only when set (common case is false; omit to save bytes).
- if (c.readonly) sink.append(",\"readonly\":true");
- if (c.minMode) sink.appendf(",\"minMode\":%u", static_cast(c.minMode)); // the mode the UI needs before it shows this
- if (c.numberField) sink.append(",\"numberField\":true"); // render a plain number input, not a slider
- if (c.hex) sink.append(",\"hex\":true"); // that number input reads and writes hexadecimal
- if (c.switchRow) sink.append(",\"switchRow\":true");
- if (c.displayStrip) sink.append(",\"displayStrip\":true");
- // The target rides with all three surface kinds, since a switch drives something too (switch1 is the global on/off) and its popup should say so.
- if (c.fader || c.encoder || c.switchRow) {
- if (c.fader) sink.append(",\"fader\":true");
- else if (c.encoder) sink.append(",\"encoder\":true");
- if (c.surfaceTarget) { sink.append(",\"target\":"); sink.writeJsonString(c.surfaceTarget); }
- }
- // An editable List shows add/delete/reorder and inline row editors; its rows carry a stable id and field descriptors.
- if (c.type == ControlType::List) {
- const auto* ls = static_cast(c.ptr);
- if (ls && ls->isEditableList()) sink.append(",\"editable\":true");
- if (ls && ls->listRowsFixed()) sink.append(",\"fixedRows\":true");
- if (ls && ls->listAsPads()) {
- sink.append(",\"pads\":true");
- const uint8_t gc = ls->listGridCols(), gr = ls->listGridRows();
- if (gc && gr) sink.appendf(",\"gridCols\":%u,\"gridRows\":%u",
- static_cast(gc), static_cast(gr));
- }
+void HttpServerModule::writeControl(JsonSink& sink, const ControlDescriptor& c) {
+ // Common wrapper for every control; per-type VALUE + EXTRAS live in Control.cpp, but Password is handled here since its API form is XOR-obfuscated + base64 and writeControlValue stays sink-neutral.
+ sink.appendf("{\"name\":\"%s\",\"type\":\"%s\",\"value\":",
+ c.name, controlTypeName(c.type));
+ if (c.type == ControlType::Password) {
+ // Not readable at a glance in `curl /api/state`, and the UI's hold-to-peek reverses it.
+ writeObfuscatedPassword(sink, static_cast(c.ptr));
+ } else {
+ writeControlValue(sink, c);
+ }
+ writeControlMetadata(sink, c);
+ // Emit optional flags only when set (common case is false; omit to save bytes).
+ if (c.readonly) sink.append(",\"readonly\":true");
+ if (c.minMode) sink.appendf(",\"minMode\":%u", static_cast(c.minMode)); // the mode the UI needs before it shows this
+ if (c.numberField) sink.append(",\"numberField\":true"); // render a plain number input, not a slider
+ if (c.hex) sink.append(",\"hex\":true"); // that number input reads and writes hexadecimal
+ if (c.switchRow) sink.append(",\"switchRow\":true");
+ if (c.displayStrip) sink.append(",\"displayStrip\":true");
+ // The target rides with all three surface kinds, since a switch drives something too (switch1 is the global on/off) and its popup should say so.
+ if (c.fader || c.encoder || c.switchRow) {
+ if (c.fader) sink.append(",\"fader\":true");
+ else if (c.encoder) sink.append(",\"encoder\":true");
+ if (c.surfaceTarget) { sink.append(",\"target\":"); sink.writeJsonString(c.surfaceTarget); }
+ }
+ // An editable List shows add/delete/reorder and inline row editors; its rows carry a stable id and field descriptors.
+ if (c.type == ControlType::List) {
+ const auto* ls = static_cast(c.ptr);
+ if (ls && ls->isEditableList()) sink.append(",\"editable\":true");
+ if (ls && ls->listRowsFixed()) sink.append(",\"fixedRows\":true");
+ if (ls && ls->listAsPads()) {
+ sink.append(",\"pads\":true");
+ const uint8_t gc = ls->listGridCols(), gr = ls->listGridRows();
+ if (gc && gr) sink.appendf(",\"gridCols\":%u,\"gridRows\":%u",
+ static_cast(gc), static_cast(gr));
}
- sink.append(c.hidden ? ",\"hidden\":true}" : "}");
}
+ sink.append(c.hidden ? ",\"hidden\":true}" : "}");
}
// Apply-core: set one control's value from the same `{"value":8}` body the HTTP handler gets; transport-free, returning an OpResult the caller maps to its own reporting.
@@ -2424,6 +2571,12 @@ void HttpServerModule::handleFirmwareUrl(platform::TcpConnection& conn, const ch
"{\"error\":\"url must start with http:// or https://\"}");
return;
}
+ // A device without TLS has no firmware fetcher at all, so it says so before starting: the UI then fetches the image itself and uploads it.
+ if (!platform::httpsAvailable()) {
+ sendResponse(conn, 501, "application/json",
+ "{\"error\":\"this device cannot fetch a URL: upload the image\"}");
+ return;
+ }
// A MoonBase device runs from its one app slot, so stage the URL in NVS and reboot into MoonBase to install; 202 with {"moonbase":true} marks this flow.
if (platform::otaHasMoonBase() && !platform::otaRunningMoonBase()) {
@@ -2520,10 +2673,11 @@ void HttpServerModule::handleWebSocketUpgrade(platform::TcpConnection& conn, con
if (!wsClients_[i].valid()) {
wsOnAccessPoint_[i] = onAccessPoint(conn); // read once: a socket's peer does not change
wsClients_[i] = std::move(conn);
- // A full state mid-drain is what this newcomer needs too: its cursor starts at 0, so it receives the whole in-flight message with no splice or cancel.
- stateSend_.sent[i] = 0;
- // A new client needs the full state, not a patch against a baseline it never received; the global cache resyncs everyone, which is cheap since connects are rare.
- requestFullResync();
+ // A fresh slot: no stream, and no state until the client subscribes, which requests the resync its first document answers.
+ endStream(streams_[i]);
+ subs_[i] = Subscription{};
+ resyncClient_[i] = false;
+ resyncId_[i] = 0;
return;
}
}
@@ -2538,39 +2692,45 @@ void HttpServerModule::pushStateToWebSockets() {
}
if (!hasClients) return;
- if (fullResyncPending_) {
- // Full state (~30 KB) drains in chunks on tick20ms, not a blocking write; a prior one finishes first, and fullResyncPending_ stays true until a start is accepted.
- if (stateSend_.active) return;
- JsonSink sink;
- buildStateJson(sink);
- // A sink out of heap holds a truncated document; send it anyway and clear the resync flag, since returning early freezes the whole UI where a partial tree keeps updating.
- if (sink.overflowed()) {
- setStatus("state too large for free memory: some modules may not show", Severity::Warning);
- }
- const size_t len = sink.size();
- char* owned = sink.detach(); // move ownership to the sender (frees on drain-complete)
- if (owned && startBufferedTextSend(owned, len)) {
- baselineLeafHashes(); // the full state IS the new baseline: next tick patches from here
- fullResyncPending_ = false; // cleared only on a confirmed accept; a failed start retries
- }
- } else {
- // PATCH, the steady state: only changed leaves (~1-2 KB) go out inline; it waits while a full state drains, since a small frame would interleave inside that WS message.
- if (stateSend_.active) return;
- JsonSink sink;
- if (buildStatePatch(sink) > 0) sendPatch(sink);
- // changed == 0 → nothing to send this second (an idle device); the common quiet case.
+ // Nothing may land inside a streamed document, so every push waits while one is open; a resync requested meanwhile starts once they end.
+ if (anyStreamActive()) return;
+ if (fullResyncPending_ || anyClientResync()) {
+ // A resync of some clients first patches the rest up to now, since the baseline taken next forgets what they have not been sent.
+ if (!fullResyncPending_) sendPatches([&](auto&& fn) { forEachStateLeaf(fn); }, /*skipResyncing=*/true);
+ baselineLeafHashes(); // the full state IS the new baseline: next tick patches from here
+ startStreams();
+ return; // the WLED frame waits for the streams too
}
+ // PATCH, the steady state: only changed leaves go out inline, each client its share; an idle device sends nothing.
+ sendPatches([&](auto&& fn) { forEachStateLeaf(fn); });
// Push a WLED-shaped {state, info} frame for the WLED app on this /ws; each consumer ignores the other's frame (the UI keys on `modules`, the app on `state`/`info`).
pushWledStateToWebSockets();
}
+// Each client due a document gets a cursor that streams it on tick20ms, a fragment at a time, so no board needs the whole tree in one block.
+void HttpServerModule::startStreams() {
+ for (int i = 0; i < MAX_WS_CLIENTS; i++) {
+ const bool wanted = fullResyncPending_ || resyncClient_[i];
+ resyncClient_[i] = false;
+ if (!wanted || !wsClients_[i].valid() || !subs_[i].received) continue; // the WLED app never subscribes, and reads its own frame
+ StateStream& s = streams_[i];
+ endStream(s);
+ s.sub = subs_[i];
+ s.cursor.sub = &s.sub;
+ s.cursor.answers = resyncId_[i];
+ resyncId_[i] = 0;
+ s.active = true;
+ }
+ fullResyncPending_ = false;
+}
+
// Build and push the WLED {state, info} object to every WS client. Shares the same body writers as /json/si.
void HttpServerModule::pushWledStateToWebSockets() {
- if (stateSend_.active) return; // never interleave with the chunked full-state drain
+ if (anyStreamActive()) return; // never interleave with a streamed full state
bool hasClients = false;
- for (auto& ws : wsClients_) if (ws.valid()) { hasClients = true; break; }
- if (!hasClients) return;
+ for (int i = 0; i < MAX_WS_CLIENTS; i++) if (wsClients_[i].valid() && !subs_[i].received) { hasClients = true; break; }
+ if (!hasClients) return; // only the web interface is connected: the frame is built for nobody
const char* name; uint8_t mac[6]; uint8_t ip[4];
resolveWledIdentity(name, mac, ip);
@@ -2583,67 +2743,113 @@ void HttpServerModule::pushWledStateToWebSockets() {
writeWledInfoBody(sink, name, mac);
sink.appendf("}");
- for (auto& ws : wsClients_) {
- if (!ws.valid()) continue;
+ for (int i = 0; i < MAX_WS_CLIENTS; i++) {
+ auto& ws = wsClients_[i];
+ // A client that subscribed is the web interface, which reads module state rather than this frame.
+ if (!ws.valid() || subs_[i].received) continue;
if (!sendWsTextFrame(ws, sink.data(), static_cast(sink.size()))) ws.close();
}
}
// Read one frame per client and apply a WLED state-set ({on}/{bri}) to Drivers; client frames are masked (RFC 6455 §5.3), so unmask in place and skip non-text frames.
-void HttpServerModule::pollWledStateFromWebSockets() {
- // Reap cleanly closed preview clients, whose slot otherwise counts against the cap until a send fails: read() == 0 is a peer FIN; no lease is needed on this core.
- for (int i = 0; i < MAX_PREVIEW_CLIENTS; i++) {
- auto& pc = previewClients_[i];
- if (!pc.valid()) continue;
- uint8_t buf[64];
- const int n = pc.read(buf, sizeof(buf));
- if (n == 0) { // clean close (peer FIN): free the slot NOW
- pc.close();
- if (clientSink_) clientSink_->onClientGone(i);
- continue;
- }
- if (n > 0 && clientSink_) {
- // Walk the read, since TCP coalesces several small requests into one buffer; each complete frame's unmasked payload goes to the sink in order.
- for (int off = 0; off < n; ) {
- uint8_t payload[8];
- int used = 0;
- const int len = parsePreviewUplink(buf + off, n - off, payload, &used);
- if (used <= 0) break; // nothing parseable left (or a partial tail)
- if (len > 0) clientSink_->onClientMessage(i, payload, len);
- off += used;
- }
- }
+void HttpServerModule::pollUplinks() {
+ for (int i = 0; i < MAX_PREVIEW_CLIENTS; i++)
+ if (previewClients_[i].valid()) pollPreviewClient(i);
+ for (int c = 0; c < MAX_WS_CLIENTS; c++) pollControlClient(c);
+}
+
+void HttpServerModule::pollPreviewClient(int i) {
+ auto& pc = previewClients_[i];
+ uint8_t buf[64];
+ const int n = pc.read(buf, sizeof(buf));
+ // A clean close frees the slot now, which otherwise counts against the cap until a send fails; no lease is needed on this core.
+ if (n == 0) {
+ pc.close();
+ if (clientSink_) clientSink_->onClientGone(i);
+ return;
+ }
+ if (n <= 0 || !clientSink_) return;
+ // TCP coalesces small requests into one read, so each complete frame's payload goes to the sink in order.
+ for (int off = 0; off < n; ) {
+ uint8_t payload[8];
+ int used = 0;
+ const int len = parsePreviewUplink(buf + off, n - off, payload, &used);
+ if (used <= 0) break; // nothing parseable left (or a partial tail)
+ if (len > 0) clientSink_->onClientMessage(i, payload, len);
+ off += used;
}
+}
- for (int c = 0; c < MAX_WS_CLIENTS; c++) {
- auto& ws = wsClients_[c];
- if (!ws.valid()) { if (carryClient_ == c) carryLen_ = 0; continue; }
- uint8_t f[768];
- size_t have = 0;
- // A frame the last read cut in half starts this one, so the stream never loses its framing.
- if (carryClient_ == c && carryLen_) { std::memcpy(f, carry_, carryLen_); have = carryLen_; carryLen_ = 0; }
- const int n = ws.read(f + have, sizeof(f) - have); // non-blocking: -1 when nothing is pending
- // A clean peer close (FIN) frees the slot now, so a reconnect burst finds it free.
- if (n == 0) { ws.close(); continue; }
- const size_t total = have + static_cast(n > 0 ? n : 0);
- if (total == 0) continue;
- const size_t used = walkWsTextFrames(f, total, [&](const char* body) {
- // A control write from the interface's input bridges, which keeps them off a TCP connection per write.
- if (mm::json::hasKey(body, "module") && mm::json::hasKey(body, "control")) {
- char moduleName[32] = {}, controlName[32] = {};
- mm::json::parseString(body, "module", moduleName, sizeof(moduleName));
- mm::json::parseString(body, "control", controlName, sizeof(controlName));
- applySetControl(moduleName, controlName, body);
- } else if (mm::json::hasKey(body, "on") || mm::json::hasKey(body, "bri") || mm::json::hasKey(body, "ps")) {
- applyWledState(body); // the WLED app's {on, bri, ps}
- }
- });
- if (used < total && total - used <= sizeof(carry_)) {
- std::memcpy(carry_, f + used, total - used);
- carryLen_ = total - used;
- carryClient_ = c;
- }
+void HttpServerModule::pollControlClient(int c) {
+ auto& ws = wsClients_[c];
+ if (!ws.valid()) {
+ if (carryClient_ == c) carryLen_ = 0;
+ if (subs_[c].received) subs_[c] = Subscription{}; // a gone client stops widening what the patch walks
+ return;
+ }
+ // Mid-fragment the client's frames wait in the socket, so whatever they answer, a Pong or a Close, goes out between frames.
+ if (!atFrameBoundary(c)) return;
+ uint8_t f[768];
+ size_t have = 0;
+ // A frame the last read cut in half starts this one, so the stream never loses its framing.
+ if (carryClient_ == c && carryLen_) { std::memcpy(f, carry_, carryLen_); have = carryLen_; carryLen_ = 0; }
+ const int n = ws.read(f + have, sizeof(f) - have); // non-blocking: -1 when nothing is pending
+ if (n == 0) { ws.close(); return; } // a clean close frees the slot for a reconnect burst
+ const size_t total = have + static_cast(n > 0 ? n : 0);
+ if (total == 0) return;
+ const size_t used = walkWsFrames(f, total, [&](const char* body) { onTextFrame(c, body); },
+ [&](uint8_t opcode, const uint8_t* payload, size_t len) { onControlFrame(c, opcode, payload, len); });
+ if (used < total && total - used <= sizeof(carry_)) {
+ std::memcpy(carry_, f + used, total - used);
+ carryLen_ = total - used;
+ carryClient_ = c;
+ }
+}
+
+void HttpServerModule::onTextFrame(int c, const char* body) {
+ using mm::json::hasKey;
+ if (hasKey(body, "module") && hasKey(body, "control")) {
+ // A control write from the interface's input bridges, which keeps them off a TCP connection per write.
+ char moduleName[32] = {}, controlName[32] = {};
+ mm::json::parseString(body, "module", moduleName, sizeof(moduleName));
+ mm::json::parseString(body, "control", controlName, sizeof(controlName));
+ applySetControl(moduleName, controlName, body);
+ } else if (hasKey(body, "subscribe")) {
+ applySubscription(c, body); // what this client views
+ } else if (hasKey(body, "resync")) {
+ // The page asks for its view again after an add, delete or move, and the document echoes the id.
+ if (!subs_[c].received) return;
+ resyncClient_[c] = true;
+ resyncId_[c] = static_cast(mm::json::parseInt(body, "resync"));
+ } else if (hasKey(body, "on") || hasKey(body, "bri") || hasKey(body, "ps")) {
+ applyWledState(body); // the WLED app's {on, bri, ps}
+ }
+}
+
+bool HttpServerModule::atFrameBoundary(int client) const {
+ const StateStream& s = streams_[client];
+ return !s.frame || s.sent == 0;
+}
+
+void HttpServerModule::onControlFrame(int client, uint8_t opcode, const uint8_t* payload, size_t len) {
+ // Read only at a frame boundary, so the answer goes out between frames, where a control frame may.
+ auto& ws = wsClients_[client];
+ if (opcode == 0x9) {
+ if (!sendWsControlFrame(ws, 0xA, payload, len)) ws.close();
+ return;
}
+ // A Close is echoed with its status code and the connection ends; a one-byte body carries no code (§5.5.1), so it is answered empty.
+ sendWsControlFrame(ws, 0x8, payload, len >= 2 ? 2 : 0);
+ ws.close();
+ endStream(streams_[client]);
+}
+
+bool HttpServerModule::sendWsControlFrame(platform::TcpConnection& conn, uint8_t opcode, const uint8_t* payload, size_t len) {
+ uint8_t frame[2 + kWsControlMax];
+ frame[0] = static_cast(0x80 | opcode);
+ frame[1] = static_cast(len);
+ if (len) std::memcpy(frame + 2, payload, len);
+ return conn.write(frame, 2 + len);
}
bool HttpServerModule::sendWsTextFrame(platform::TcpConnection& conn, const char* data, int len) {
@@ -2718,20 +2924,6 @@ bool HttpServerModule::sendBufferedFrame(const uint8_t* header, size_t headerLen
return true;
}
-// Queue a text frame whose body this module owns through the same resumable slot, so the state JSON drains in chunks on tick20ms, not a blocking write.
-bool HttpServerModule::startBufferedTextSend(char* ownedBody, size_t bodyLen) {
- // A send already in flight: drop this one and free its buffer: the next second's state is fresher.
- if (stateSend_.active) { platform::free(ownedBody); return false; }
- // No app header for the state frame (the JSON is the whole payload), just the WS text header.
- const size_t wsLen = writeWsFrameHeader(stateSend_.hdr, 0x81, bodyLen);
- stateSend_.hdrLen = wsLen;
- stateSend_.body = reinterpret_cast(ownedBody);
- stateSend_.bodyLen = bodyLen;
- for (auto& c : stateSend_.sent) c = 0;
- stateSend_.active = true;
- return true; // drained on tick20ms, never a blocking write on the render tick
-}
-
// Per-client cursor over [hdr ++ body]: write what the socket takes (one chunk), advance, resume next tick; an error closes that client, the send ends when every client is done.
void HttpServerModule::drainPreviewSend() {
// Core-0 side of the sender lease core 1 holds while arming a frame; try_lock, not a wait, since tick20ms must not block, so we drain next tick.
@@ -2771,38 +2963,89 @@ void HttpServerModule::drainPreviewSend() {
if (!anyLiveClient || allDone) previewSend_.active = false;
}
-// The same cursor drain for the full-state frame over the control clients; core-0 only, so unlike the preview slot it needs no sender lease.
+void HttpServerModule::applySubscription(int client, const char* body) {
+ if (client < 0 || client >= MAX_WS_CLIENTS) return;
+ Subscription next;
+ if (!next.parse(body) || next == subs_[client]) return;
+ subs_[client] = next;
+ resyncClient_[client] = true; // this client alone: the push brings the others up to date before it re-baselines
+}
+
+bool HttpServerModule::anyClientResync() const {
+ for (bool r : resyncClient_)
+ if (r) return true;
+ return false;
+}
+
+bool HttpServerModule::anyStreamActive() const {
+ for (const auto& st : streams_)
+ if (st.active) return true;
+ return false;
+}
+
+void HttpServerModule::endStream(StateStream& s) {
+ if (s.frame) platform::free(s.frame);
+ s = StateStream{};
+}
+
+bool HttpServerModule::nextStreamFrame(int client) {
+ StateStream& s = streams_[client];
+ JsonSink sink;
+ bool done;
+ {
+ // Per client: one on the device's own access point gets its passwords empty, the others as stored.
+ const SecretsHidden hide(wsOnAccessPoint_[client]);
+ done = writeTreeFragment(sink, s.cursor, kStreamFragmentBytes);
+ }
+ if (sink.overflowed()) return false; // a truncated fragment would end the message in invalid JSON
+ s.frameLen = sink.size();
+ s.frame = sink.detach();
+ if (!s.frame) return false;
+ // RFC 6455 §5.4: the first frame carries the text opcode, the rest continue it, and the last one sets FIN.
+ const uint8_t opcode = static_cast((s.first ? 0x01 : 0x00) | (done ? 0x80 : 0x00));
+ s.hdrLen = writeWsFrameHeader(s.hdr, opcode, s.frameLen);
+ s.sent = 0;
+ s.first = false;
+ s.last = done;
+ return true;
+}
+
+// Per client, the fragment in flight goes out a chunk at a time and the next is generated once it has left; core-0 only, so unlike the preview slot it needs no sender lease.
void HttpServerModule::drainStateSend() {
- if (!stateSend_.active) return;
- const size_t total = stateSend_.hdrLen + stateSend_.bodyLen;
const size_t chunk = drainChunkBytes();
- bool anyLiveClient = false;
- bool allDone = true;
- for (int i = 0; i < MAX_WS_CLIENTS; i++) {
- auto& ws = wsClients_[i];
- if (!ws.valid()) continue;
- anyLiveClient = true;
- size_t& cur = stateSend_.sent[i];
- size_t budget = chunk;
- while (cur < total && budget > 0) {
- const uint8_t* src;
- size_t span;
- if (cur < stateSend_.hdrLen) { src = stateSend_.hdr + cur; span = stateSend_.hdrLen - cur; }
- else { src = stateSend_.body + (cur - stateSend_.hdrLen); span = total - cur; }
- if (span > budget) span = budget;
- int n = ws.writeSome(src, span);
- if (n < 0) { ws.close(); break; } // real error, drop this client
- if (n == 0) break; // WouldBlock, resume next tick
- cur += static_cast(n);
- budget -= static_cast(n);
+ for (int i = 0; i < MAX_WS_CLIENTS; i++)
+ if (streams_[i].active) drainStream(i, chunk);
+}
+
+void HttpServerModule::drainStream(int client, size_t budget) {
+ StateStream& s = streams_[client];
+ auto& ws = wsClients_[client];
+ if (!ws.valid()) { endStream(s); return; }
+ while (budget > 0) {
+ if (!s.frame) {
+ if (s.last) { endStream(s); return; }
+ if (!nextStreamFrame(client)) { refuseFragment(client); return; }
+ fragmentRefused_ = false;
}
- if (ws.valid() && cur < total) allDone = false;
- }
- if (!anyLiveClient || allDone) {
- platform::free(const_cast(stateSend_.body)); // the frame owns its JSON body
- stateSend_.body = nullptr;
- stateSend_.active = false;
- }
+ const size_t total = s.hdrLen + s.frameLen;
+ const bool inHeader = s.sent < s.hdrLen;
+ const uint8_t* src = inHeader ? s.hdr + s.sent : reinterpret_cast(s.frame) + (s.sent - s.hdrLen);
+ const size_t span = std::min(budget, (inHeader ? s.hdrLen : total) - s.sent);
+ const int n = ws.writeSome(src, span);
+ if (n < 0) { ws.close(); endStream(s); return; } // real error, drop this client
+ if (n == 0) return; // WouldBlock, resume next tick
+ s.sent += static_cast(n);
+ budget -= static_cast(n);
+ if (s.sent == total) { platform::free(s.frame); s.frame = nullptr; }
+ }
+}
+
+void HttpServerModule::refuseFragment(int client) {
+ // Out of memory for one fragment: the client reconnects and is sent its document again, and the serial log says why, once per run of refusals.
+ if (!fragmentRefused_) std::printf("state stream: no memory for a %u-byte fragment (largest block %u), client closed\n", static_cast(kStreamFragmentBytes), static_cast(platform::maxAllocBlock()));
+ fragmentRefused_ = true;
+ wsClients_[client].close();
+ endStream(streams_[client]);
}
// Per-client chunk cap per tick from free contiguous memory, between a floor (progress) and a ceiling (tick occupancy), so a tight board takes small bites.
diff --git a/src/core/system/HttpServerModule.h b/src/core/system/HttpServerModule.h
index 01a0b97a3..d62114275 100644
--- a/src/core/system/HttpServerModule.h
+++ b/src/core/system/HttpServerModule.h
@@ -22,10 +22,7 @@ class Scheduler;
///
/// ## What it is allowed to include
///
-/// This is core infrastructure, and includes no light header.
-/// The WLED-compatibility shim reads the device shape and the palettes as colors through `LightOutput`, which the light domain implements, and drives the palette through `Scheduler::setControl`.
-///
-/// The implementation lives in the `.cpp`, and this header is the interface alone. The `port` control defaults to 8080 on the desktop and 80 on an ESP32.
+/// This is core infrastructure and includes no light header; the WLED shim reads the device and its palettes through `LightOutput` and drives the palette through `Scheduler::setControl`.
///
/// ## The REST API
///
@@ -39,173 +36,171 @@ class Scheduler;
///
/// ## Mutations and the File Manager
///
-/// A mutation is `POST /api/control` with a module, control and value; `POST /api/modules` to create; `POST /api/modules/{name}/move` to reorder and `.../replace` to swap; `POST /api/reboot`; or `DELETE /api/modules/{name}`.
-///
-/// The File Manager reads and writes through `GET|POST /api/file?path=`, and lists, creates and removes through `/api/dir?path=`. The path rides the query, so a filesystem operation carries its target in the request rather than in a stored control.
-/// A board with an SD slot (`SdCardService`) mounts it at `/sd`, so these same two routes reach the card too, shown as a folder next to the internal flash.
+/// A mutation is `POST /api/control`, `POST /api/modules` to create, `.../move` and `.../replace`, `POST /api/reboot`, or `DELETE /api/modules/{name}`.
///
-/// Every JSON response streams through a `JsonSink`, so there is no fixed-buffer ceiling and a tree of any size serializes correctly.
+/// The File Manager reads and writes through `/api/file?path=` and lists, creates and removes through `/api/dir?path=`; a card mounted at `/sd` takes the same routes.
///
/// ## The two WebSocket channels
///
-/// `GET /ws` with an `Upgrade: websocket` header does the RFC 6455 handshake, SHA-1 and base64.
-/// Traffic is split by class across two channels sharing one lwIP socket budget, each with its own cap. `/ws` carries the control plane, the JSON state and its patches, and `/wsp` the lossy binary preview stream.
+/// `GET /ws` with `Upgrade: websocket` does the RFC 6455 handshake. `/ws` carries the JSON state and its patches, and `/wsp` the lossy binary preview, each with its own client cap.
+///
+/// A binary frame drains in memory-adaptive chunks per client per 20 ms tick. A large frame so crosses several ticks without a loop waiting on a socket, and still arrives as one message.
///
-/// Every binary message takes one path: the resumable buffered send.
-/// It drains a memory-adaptive chunk per client per 20 ms tick, from a stable caller-owned buffer. A large frame is therefore delivered across wall-clock ticks without any loop waiting on a socket, while staying one atomic WebSocket message.
-/// One buffered send is in flight per slot at a time, newest-wins: a new offer arriving while one is active is dropped. A client is closed only on a real error or a FIN, never for being slow.
+/// One frame is in flight per channel and a new offer meanwhile is dropped. A client is closed on a real error or a FIN, never for being slow.
///
/// ## What a preview client may send back
///
-/// Inbound `/wsp` payloads are unmasked and handed opaquely to the registered producer sink, whose vocabulary is a standing frame request and a one-shot table request.
-/// On `/ws` a client may write a control as `{"module","control","value"}`, which the input bridges use at up to 30 writes a second; every other mutation goes through REST.
+/// Inbound `/wsp` payloads go unmasked and opaque to the producer's sink: a standing frame request and a one-shot table request.
+///
+/// On `/ws` a client writes a control as `{"module","control","value"}`, as the input bridges do up to 30 times a second; every other mutation goes through REST.
///
/// ## State push: a diff on the wire
///
-/// This is the recognizable snapshot-then-patch model, the shape Redux and Firestore sync use and RFC 6902 formalises.
+/// Snapshot then patch, the shape Redux and Firestore sync use and RFC 6902 formalises. Re-serializing the mostly static tree every second on the render thread stuttered the LEDs at 1 Hz.
+///
+/// A client gets a full state when it subscribes, then each second a patch of the values that changed. Each leaf's serialized value is hashed with FNV-1a and compared with the cached hash.
+///
+/// Value-compare rather than a dirty flag, so a value the device changes itself, telemetry, a status or a driver, is caught like a `setControl` write with no per-write bookkeeping.
+///
+/// ## State by subscription
+///
+/// Publish/subscribe per client: `{"subscribe":[…]}` names the modules a client views, by name or type, each with its descendants, and `"*"` names the whole tree.
+///
+/// The full state carries those modules whole and every other module as an outline: its header and children, without controls. A page so gets its root and its status bar's modules.
+///
+/// A connect alone sends nothing, so a page that subscribes on open gets one document, and the WLED app, which never subscribes, gets only its own frame.
+///
+/// A subscribe resyncs that client alone. The others first get the patch due, then the baseline is retaken, so none misses a change; a schema change resyncs everyone.
+///
+/// `{"resync":n}` asks for the view again, as the page does after an add, delete or move. The answer ends with `"resync":n`, so a document already under way is not taken for it.
+///
+/// ## Ping and Close
+///
+/// A Ping gets a Pong with its payload, and a Close a Close with its status code before the connection ends (RFC 6455 §5.5). A browser sends neither; a client library pings to keep alive.
+///
+/// The uplink is read only between frames, so the answer can never land inside a fragment, where RFC 6455 §5.4 forbids it.
+///
+/// ## The full state streams
+///
+/// Each client's document streams from its own cursor, a depth-first position resumable at a module's header, each control and each child, a fragment of about 2 KB at a time.
+///
+/// The first fragment opens a text message, the rest continue it and the last carries FIN: the fragmentation RFC 6455 §5.4 gives a message of unknown size.
+///
+/// A board so needs one fragment, or its largest control, in one block. The whole tree is more than a 1 MB board's heap can hand out at once.
///
-/// The state a client needs is the full module tree, around 30 KB and mostly unchanging option and detail metadata. Re-serializing all of it every second, inline on the render thread, stole render budget and stuttered the LEDs at 1 Hz.
+/// The cursor resolves its path again at every step, so a tree changed between fragments still ends in valid JSON, and the schema change queues the resync that follows.
///
-/// So a client gets the full state once on connect, chunk-drained through the resumable sender and off the render tick.
-/// Each second after that it gets a patch carrying only the values that changed. Change is found by value-compare rather than a dirty flag: each leaf's value is serialized, hashed with FNV-1a, and compared against a cached hash.
-/// A value the device mutates itself, telemetry or a status or a driver, is therefore caught exactly like a `setControl` write. No per-write instrumentation is needed.
+/// Run to completion, the same cursor writes `/api/state` and one module's JSON. Patches and the WLED frame wait while any document streams.
///
/// ## A value someone watches move
///
-/// A module whose value moves in front of someone, a desk's motorized faders, calls `notifyValuesChanged()`. Its changed leaves then go out on the next 20 ms tick in a patch of their own, at most every 40 ms, rather than on the next second.
-/// Only that module is walked, and the hashes update as it goes, so the second's patch does not send the values again.
+/// A module whose values move in front of someone, a desk's motorized faders, calls `notifyValuesChanged()`. Its changes go out on the next 20 ms tick, at most every 40 ms.
+///
+/// Only that module is walked and its hashes update as it goes, so the second's patch does not send the values again.
///
/// ## How a leaf is named
///
-/// A leaf path is the module name and the control name, or an `@`-prefixed field for live per-card header telemetry. Module names are unique tree-wide, so a path is stable.
+/// A leaf path is the module name and the control name, or an `@`-prefixed field for per-card header telemetry. Module names are unique tree-wide, so a path is stable.
///
/// ## When a patch cannot describe the change
///
-/// The hash cache is one global baseline rather than one per client, held in a growable scratch buffer.
-/// A full resync re-sends the whole state and re-baselines. That happens on connect and after any structural change, since a value patch cannot describe a reshaped tree.
-///
-/// A schema change forces one too, through a static schema-changed hook. The value patch carries no changed hidden flags or option sets. Any `rebuildControls` triggers it: a control set, a list mutation, an asynchronous WiFi or Hue callback.
+/// One baseline covers what any client subscribes to. A patch is collected once over it, one slice per module, and each client gets the slices its subscription covers.
///
-/// A pending resync is fast-pathed on the 20 ms tick rather than only the 1 s one, and preempts an in-flight preview frame.
-/// A freshly connected client therefore gets its state, and so its preview, within tens of milliseconds rather than up to a second.
+/// A full resync resends the state and retakes the baseline, on a subscribe and after any structural change, since a value patch cannot describe a reshaped tree.
///
-/// ## What the diff buys
+/// A schema change forces one through a static hook, since a patch carries no hidden flags or option sets. Any `rebuildControls` triggers it: a control set, a list edit, a network callback.
///
-/// The net effect is a per-second push falling from about 34 KB to one or two. The expensive full-tree serialize runs only on connect or a schema change.
-/// The UI applies a patch in place, with no rebuild, and re-renders on a full frame. That is the sub-hot-path rule applied: a periodic tick shares the render thread, so its work must be cheap or skipped when nothing changed.
+/// A pending resync runs on the 20 ms tick and preempts an in-flight preview frame, so a new client gets its state, and its preview, within tens of milliseconds.
///
/// ## Why the drain runs on the transport tick
///
-/// The resumable drain runs on the 20 ms transport poll and deliberately not the per-render `tick`. Pushing preview bytes to a socket is therefore never charged to the LED render hot path.
-/// The LED output is never delayed by the preview. The preview frame rate is instead bounded by the 20 ms drain cadence, which is the right trade: the preview is a view and the LEDs are not.
+/// The drain runs on the 20 ms transport poll, not the per-render `tick`, so a socket write is never charged to the LEDs. The preview's rate is bounded by that cadence: it is a view, the LEDs are not.
///
/// ## The WLED-compatibility shim
///
-/// A small set of WLED-shaped messages make a MoonLight device appear in the native WLED apps and Home Assistant's WLED integration, and be controlled from them.
+/// WLED-shaped messages make a MoonLight device appear in the WLED apps and Home Assistant's WLED integration, and be controlled from them. Discovery is over mDNS.
///
-/// Discovery is over mDNS.
-/// Validation is a minimal `GET /json/info` carrying a name, a MAC, LED and WiFi objects, and the brand and product fields. The app keys on the brand to accept a device: we interoperate rather than impersonate, and this is no full WLED emulation.
+/// `GET /json/info` carries a name, a MAC, LED and WiFi objects and the brand and product fields. The app keys on the brand: we interoperate rather than impersonate.
///
-/// Live state is pushed over `/ws` as a state-and-info frame. The state mirrors the Drivers brightness control, and its color is the active palette's, from `LightOutput::paletteRgb()`, or black on a device with no light output.
+/// Live state goes over `/ws` as a state-and-info frame: Drivers brightness, and the active palette's color from `LightOutput::paletteRgb()`, or black without a light output.
///
-/// Control is bidirectional over that same socket. The app's slider and toggle send a frame that is read and applied to Drivers brightness through the shared apply core, the path REST and Improv also use.
+/// The app's slider and toggle send a frame on the same socket, applied to Drivers brightness through the apply core that REST and Improv share.
///
/// ## Reading an uplink frame, and taking the lease
///
-/// The uplink parser handles RFC 6455 framing alone, and the payload's meaning belongs to the registered sink. It returns a payload length, or -1 when the buffer holds no complete frame.
-/// It also reports how many bytes the whole frame consumed, so a caller can walk a read that coalesced several. It is pure and static, so the byte handling is unit-tested without a socket.
+/// The uplink parser handles RFC 6455 framing alone. It returns a payload length, or -1 without a complete frame, and the bytes consumed, so a coalesced read walks frame by frame.
+///
+/// It is pure and static, so the byte handling is unit-tested without a socket.
///
-/// The sender lease guards the send state and the socket writes against this module's own core-0 drain and state push, while an offloaded `PreviewDriver` streams from core 1.
-/// It try-locks rather than waits: a busy transport returns false and the producer skips that frame.
+/// The sender lease guards the preview's send state against this module's core-0 drain while an offloaded `PreviewDriver` streams from core 1. It try-locks: a busy transport skips a frame.
///
/// ## The per-module state route
///
-/// One module's JSON is served on its own route, byte-identical to that module's entry in the full state, children included.
-/// An issue report carries it from the card's `{ }` popup, and `/document` on the same route serves the module as the state document a preset holds. A name in the path may be percent-encoded, since a module name can carry a space.
+/// One module's JSON, children included, is served on its own route, byte-identical to its entry in the full state; `/document` serves it as a preset's state document. A name may be percent-encoded.
///
/// ## Two firmware install paths
///
/// A firmware URL is answered at once and installed on its own task, so the browser polls for progress rather than holding a request open.
-/// On a MoonBase device the URL is staged in NVS and the device reboots into MoonBase, which installs it unattended; otherwise the platform fetches straight to the OTA slot.
///
-/// Installing a new MoonBase into the factory slot is the mirror of that, and the only way to fix a broken recovery image without a cable.
-/// It vets the image before erasing anything and deliberately does not reboot, so the running app is untouched. The URL form of it is what makes a release asset installable without the browser relaying most of a megabyte.
+/// On a MoonBase device the URL is staged in NVS and the device reboots into MoonBase, which installs it; otherwise the platform fetches straight to the OTA slot.
+///
+/// Installing a MoonBase into the factory slot mirrors that, and is the only cable-free fix for a broken recovery image. It vets the image before erasing and does not reboot.
///
/// ## Crossing the domain boundary
///
-/// This module exposes the `BinaryBroadcaster` interface, and the light-domain `PreviewDriver` holds a pointer to it and streams each frame's bytes through it.
-/// `main.cpp` wires the two together, being the only file that knows both, and the preview's point budget and wire format are the driver's concern.
+/// The light-domain `PreviewDriver` streams each frame through this module's `BinaryBroadcaster` interface. `main.cpp`, the one file that knows both, wires them together.
///
/// ## The apply core: the REST API, callable in process
///
-/// The add, set and clear-children operations are factored out of the connection so any transport can drive them.
-/// Two do today: the HTTP handlers, thin wrappers mapping a result onto a status code, and the Improv serial path, which applies a pushed op on the main loop.
-/// That gives the apply logic one home, and transports differ only in how they frame a request and report its result.
+/// The add, set and clear-children operations are factored out of the connection, so the HTTP handlers and the Improv serial path share one apply logic and differ only in framing.
///
-/// A file write is the second way persistent state changes, so it ends in the same request that a control write does.
-/// Putting it here rather than in whichever client remembers to send a follow-up nudge is what makes it reliable, since `curl` and MoonDeck would not.
+/// A file write ends in the same re-derive request a control write does, here rather than in a client that remembers a follow-up nudge, which `curl` and MoonDeck would not.
///
-/// The re-derive asks the whole tree rather than consulting a path-to-module registry, which would need an association nothing else in the system keeps.
-/// A module decides for itself whether what it holds actually changed. `prepare` is the cold path that exists to be re-entered, and a scripted module compares a content hash rather than re-reading its source.
-/// The path is accepted for diagnostics and a future narrower dispatch; today every successful write asks the same question.
+/// The re-derive asks the whole tree, and each module decides whether what it holds changed: a scripted module compares a content hash rather than re-reading its source.
///
/// ## Two guards that are static on purpose
///
-/// The filesystem path decoder is the single guard every filesystem entry shares, for reads, writes, listings, directory creation and deletion. It refuses a missing or empty path, a `..` traversal, and an overlong value that would fill the buffer.
-///
-/// The header lookup is case-insensitive because RFC 9112 says field names are. Browsers send one spelling and node's client another, and the case-sensitive search this replaced silently read a length of zero and committed empty files with a 200.
+/// The path decoder is the one guard every filesystem route shares: it refuses a missing or empty path, a `..` traversal and an overlong value. Both guards are static, unit-tested without a socket.
///
-/// Both are public and static so the byte handling is unit-tested without a socket fixture.
+/// The header lookup is case-insensitive, as RFC 9112 says field names are. A case-sensitive search reads node's lowercase `content-length` as zero and writes empty files.
///
/// ## What the preview lock guards, and what it does not
///
-/// The lock covers the preview channel's shared state, its send bookkeeping and its sockets, because those have producers on two cores once the multicore split engages.
-/// Core 1 arms frames and streams the coordinate table from the offloaded driver's tick, while core 0 touches the same sockets in the drain, the uplink reap and admission.
+/// The lock covers the preview channel's send state and sockets, which have producers on both cores once the multicore split engages: core 1 arms frames, core 0 drains and reaps.
///
-/// Three things go wrong without it. A partial write on one core interleaves with the other inside a single frame and corrupts the framing. A close lands under a concurrent write on the same descriptor. Or one side observes torn send state.
+/// Without it a partial write on one core interleaves inside the other's frame, a close lands under a concurrent write, or one side reads torn send state.
///
-/// The control channel is deliberately outside its scope, since every writer there runs on core 0.
-/// It try-locks and never blocks: whichever core loses the race skips its slot, which the hot-path rule requires. A lost race costs one preview frame, or defers a reap or an admission by a tick, and never stalls a render or an encode.
+/// The control channel is outside it, every writer there running on core 0. The lock try-locks: the loser skips a slot, costing a preview frame and never stalling a render.
///
/// ## The served port is not the port control
///
-/// `servedPort` reports what the listener actually bound, or zero when none is up, and it is the one true source for any module that must print its own URL. The `port` control is mutable and says what was asked for, which is a different question.
+/// `servedPort` reports the port the listener bound, or zero, and is what a module prints in its URL; the `port` control says what was asked for.
///
-/// The schema hook can be installed without opening the listener.
-/// A unit test proving the hook fires a resync needs no socket, and binding a port under test is flaky when that port is busy. A release unwires it exactly as it would after a real setup.
+/// The schema hook installs without the listener, so a unit test proves it fires a resync without binding a port that may be busy.
///
/// ## How the two channel caps are sized
///
-/// The control channel allows eight clients, sized for a few concurrent viewers plus the transient overlap when one refreshes.
-/// A browser opens the new socket before the OS delivers the old one's FIN, so both briefly hold a slot. The dead one is reaped within a tick or two, on its next failed send.
-/// At four, a couple of quick refreshes filled every slot with not-yet-reaped connections and the new upgrade was rejected outright. Eight leaves headroom so a refresh always lands a slot, and a slot is only a descriptor and a small cursor.
+/// The control channel takes eight clients: a few viewers plus a refresh's overlap. The new socket opens before the old one's FIN arrives, and holds a slot until the old one is reaped.
+///
+/// At four, quick refreshes fill every slot with dead connections and the new upgrade is refused. A slot is only a descriptor and a stream cursor.
///
/// ## Why the preview cap is lower
///
-/// The preview channel allows four, deliberately fewer. Both arrays draw on one lwIP socket budget of sixteen, shared with HTTP, mDNS, Art-Net, MQTT and OTA.
-/// A preview socket per control client would consume the whole budget at the cap and starve the rest. Four is sized from observed use: one browser almost always, two often enough that it must simply work, more only occasionally.
-/// A REST caller such as Home Assistant never opens a preview socket at all. A refused upgrade costs that client only its preview, leaving its control connection untouched.
+/// The preview takes four. Both channels share lwIP's sixteen sockets with HTTP, mDNS, Art-Net, MQTT and OTA, so a preview per control client would starve the rest.
///
-/// ## Why the preview has its own connections
+/// One browser is usual and two must work. A refused preview costs that client its preview alone, and a REST caller such as Home Assistant opens none.
///
-/// They are separate TCP connections, so a 10 KB preview frame can never delay a state push. That is the standard remedy for the head-of-line blocking one socket carrying both traffic classes produces.
+/// ## Why the preview has its own connections
///
-/// The state push has its own send slot for the same reason. Sharing the preview's routed the full state to the preview channel and starved every control client of its resync.
-/// The preview's body is borrowed rather than owned, the producer keeping its pixel buffer alive and cancelling before a resize frees it.
-/// The header buffer is 24 bytes and not 16: a payload over 64 KB takes the ten-byte length form, and the app headers add eleven more. At 16, every uncapped-size frame was silently refused.
+/// Separate TCP connections keep a 10 KB preview frame from delaying a state push, the standard remedy for head-of-line blocking. The state push has its own streams for the same reason.
///
/// ## Why the sink helpers are members
///
-/// The `JsonSink` helpers below are private members rather than free functions, because each reads this module's own state or calls another of its members.
-///
-/// Three pieces live in their own headers, all in the same namespace so the call sites are unchanged.
-/// They are the sink itself with its escaping writer, the SHA-1 of RFC 3174 for the handshake, and the base64 encoder the handshake and password obfuscation share.
+/// The `JsonSink` helpers are members because each reads this module's state. The sink, the SHA-1 of RFC 3174 and the base64 encoder live in headers of their own.
///
/// ## Prior art
///
-/// The shim's exact field requirements were reverse-engineered from the [WLED-Android client](https://github.com/Moustachauve/WLED-Android) by Christophe Gagnier.
-/// Its mDNS browse, its `/json/info` validation with a non-empty MAC check, its models and its live-state socket client told us precisely what the app reads.
-/// Knowing that is why the shim is the minimal accepted object rather than a guessed full emulation.
+/// The shim's field requirements come from the [WLED-Android client](https://github.com/Moustachauve/WLED-Android) by Christophe Gagnier, so it is the minimal object that app accepts rather than a guessed full emulation.
class HttpServerModule : public MoonModule, public BinaryBroadcaster {
public:
/// The port it listens on, 8080 on the desktop and 80 on a device. @xref{the-served-port-is-not-the-port-control}
@@ -251,9 +246,15 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster {
static int parsePreviewUplink(const uint8_t* buf, int n, uint8_t out[8], int* consumed);
/// The longest text frame a client writes, a MIDI batch included. @xref{what-a-preview-client-may-send-back}
static constexpr size_t kWsTextMax = 400;
- /// Hand each complete masked text frame in `buf` to `onText`, returning how many bytes it consumed; a frame cut off at the end is left unconsumed.
- template
- static size_t walkWsTextFrames(const uint8_t* buf, size_t total, OnText&& onText) {
+ /// The longest control frame payload, a Ping's or a Close's (RFC 6455 §5.5).
+ static constexpr size_t kWsControlMax = 125;
+ /// Unmask a client frame's payload: `maskAt` points at the four-byte key, the payload following it (RFC 6455 §5.3).
+ static void unmask(const uint8_t* maskAt, size_t len, uint8_t* out) {
+ for (size_t i = 0; i < len; i++) out[i] = maskAt[4 + i] ^ maskAt[i & 3];
+ }
+ /// Hand each complete masked text frame in `buf` to `onText`, and each Ping or Close to `onControl` unmasked, returning the bytes consumed. A frame cut off at the end is left unconsumed.
+ template
+ static size_t walkWsFrames(const uint8_t* buf, size_t total, OnText&& onText, OnControl&& onControl) {
size_t off = 0;
while (off + 6 <= total) {
const uint8_t* fr = buf + off;
@@ -272,11 +273,15 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster {
const size_t frameLen = hdr + 4 + len; // header, mask key, payload
if (off + frameLen > total) break; // the rest arrives in a later read
if (opcode == 0x1 && len < kWsTextMax) {
- const uint8_t* mask = fr + hdr;
char body[kWsTextMax];
- for (size_t i = 0; i < len; i++) body[i] = static_cast(fr[hdr + 4 + i] ^ mask[i & 3]);
+ unmask(fr + hdr, len, reinterpret_cast(body));
body[len] = 0;
onText(body);
+ } else if ((opcode == 0x8 || opcode == 0x9) && len <= kWsControlMax && (fr[0] & 0x80)) {
+ // A control frame is at most 125 bytes and never fragmented (RFC 6455 §5.5); one that breaks either is malformed and skipped.
+ uint8_t payload[kWsControlMax];
+ unmask(fr + hdr, len, payload);
+ onControl(opcode, payload, len);
}
off += frameLen;
}
@@ -344,13 +349,15 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster {
/// Test seams for the wire diff, so a unit test proves it without a socket.
uint16_t buildStatePatchForTest(JsonSink& sink) { return buildStatePatch(sink); }
/// Re-baseline the leaf hashes, for a test.
- void baselineLeafHashesForTest() { baselineLeafHashes(); }
+ void baselineLeafHashesForTest() { baselineLeafHashes(true); }
/// Ask for a full resync, for a test.
void requestFullResyncForTest() { requestFullResync(); }
/// Whether a full resync is pending, for a test.
bool fullResyncPendingForTest() const { return fullResyncPending_; }
/// Clear the pending resync, for a test.
void clearFullResyncForTest() { fullResyncPending_ = false; }
+ /// Whether one client alone waits for a full state, for a test.
+ bool clientResyncPendingForTest(int client) const { return resyncClient_[client]; }
/// The port the live server is actually bound to, or 0 when none is up.
static uint16_t servedPort() { return instance_ ? instance_->boundPort_ : 0; }
@@ -427,24 +434,103 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster {
// Borrowed, not owned: the producer keeps it alive and cancels before a resize frees it.
};
PreviewSend previewSend_;
- // The same shape as PreviewSend, but draining to the control channel and owning its body.
- struct StateSend {
- uint8_t hdr[16] = {};
- size_t hdrLen = 0;
- const uint8_t* body = nullptr;
- size_t bodyLen = 0;
- size_t sent[MAX_WS_CLIENTS] = {};
+public:
+ /// What one control client views: modules named at any depth by name or type, each bringing its descendants. @xref{state-by-subscription}
+ struct Subscription {
+ static constexpr uint8_t kMaxModules = 8; ///< a page names its root and five types it reads beside it
+ bool received = false; ///< the client has subscribed, which is what its first document answers
+ bool all = false; ///< `"*"`: the whole tree
+ uint8_t count = 0; ///< how many names are held
+ uint32_t names[kMaxModules] = {}; ///< each name's FNV-1a, a module name or a type name of any length
+ /// Whether `mod` or one of its ancestors is named, by its name or its type.
+ bool covers(const MoonModule* mod) const;
+ /// Whether `s` is one of the names held.
+ bool names_(const char* s) const;
+ /// Read `{"subscribe":[…]}`, false when the body holds no such array.
+ bool parse(const char* body);
+ /// Whether two subscriptions name the same modules.
+ bool operator==(const Subscription& o) const;
+ };
+
+ /// A depth-first position in the module tree, which one serialization resumes from. @xref{the-full-state-streams}
+ struct TreeCursor {
+ static constexpr uint8_t kMaxDepth = 8; ///< deeper than any tree MoonLight builds (root, module, child, grandchild); a deeper child would end its parent's object childless
+ const Subscription* sub = nullptr; ///< null: every module in full
+ MoonModule* single = nullptr; ///< one module alone, without the `{"modules":[…]}` wrapper
+ uint8_t idx[kMaxDepth] = {}; ///< the child index at each level of the path
+ uint8_t depth = 0; ///< levels on the path, 0 before the document begins
+ uint8_t phase = 0; ///< the deepest module's part in progress: header, controls or children
+ uint8_t ctrl = 0; ///< the next control, in the controls part
+ bool wroteRoot = false; ///< a root went out, so the next one takes a comma
+ bool done = false; ///< the document has ended
+ static constexpr uint8_t kHeader = 0, kControls = 1, kChildren = 2; ///< the values of `phase`
+ uint32_t answers = 0; ///< the `resync` request the streamed state answers, echoed at its end; 0 for none
+ };
+ /// Append from `c` until the sink holds `target` bytes or the document ends, true once it has ended; 0 means no target.
+ bool writeTreeFragment(JsonSink& sink, TreeCursor& c, size_t target);
+ /// One step where the cursor's path names no module: close what is open and move on, true once the document has ended.
+ bool stepPast(JsonSink& sink, TreeCursor& c);
+ /// One step of the module at the cursor: its header, a control, or its children.
+ void stepInto(JsonSink& sink, TreeCursor& c, MoonModule* mod);
+
+ /// Set a client slot's subscription, as its `{"subscribe":…}` frame does, for a test.
+ void subscribeForTest(int client, const char* body) { applySubscription(client, body); }
+ /// Mark a client's stream open without draining it, for a test of what waits while a document is under way.
+ void holdStreamForTest(int client) { streams_[client].active = true; }
+ /// Collect the next patch once and write each of the first `count` client slots its share, for a test with no socket.
+ void buildClientPatchesForTest(JsonSink* outs, int count);
+
+private:
+ // The resumable fragment a stream sends at a time: small enough for a fragmented heap, large enough that frame headers stay noise.
+ static constexpr size_t kStreamFragmentBytes = 2048;
+ // One client's full state in flight: the cursor, and the fragment leaving as one WebSocket frame. @xref{the-full-state-streams}
+ struct StateStream {
bool active = false;
+ Subscription sub; // copied at the start, so a subscribe arriving mid-document cannot switch the view halfway
+ TreeCursor cursor;
+ char* frame = nullptr; // the fragment in flight, owned
+ size_t frameLen = 0;
+ uint8_t hdr[10] = {};
+ size_t hdrLen = 0;
+ size_t sent = 0; // over hdr ++ frame
+ bool first = true; // the next frame opens the message
+ bool last = false; // the frame in flight ends it
};
- StateSend stateSend_;
+ StateStream streams_[MAX_WS_CLIENTS];
+ Subscription subs_[MAX_WS_CLIENTS];
+ bool resyncClient_[MAX_WS_CLIENTS] = {}; // this client alone needs a full state: it subscribed, changed what it views, or asked again
+ uint32_t resyncId_[MAX_WS_CLIENTS] = {}; // the id of the request a client's next document answers
+ bool fragmentRefused_ = false; // a fragment found no memory; logged once per run of refusals
+ // Whether a frame may go to this client now: no fragment of its stream is half written. The uplink is read only then, so a Ping is always answered between frames (RFC 6455 §5.4).
+ bool atFrameBoundary(int client) const;
+ // Answer a Ping with a Pong, or a Close with a Close and the end of the connection. @xref{ping-and-close}
+ void onControlFrame(int client, uint8_t opcode, const uint8_t* payload, size_t len);
+ // Write one unfragmented control frame, `opcode` with FIN, carrying `payload`.
+ static bool sendWsControlFrame(platform::TcpConnection& conn, uint8_t opcode, const uint8_t* payload, size_t len);
// Guards the preview channel alone, whose producers sit on two cores under the split.
mutable TryLock wsLock_;
- // Queue an owned text body into the state slot, freeing it and refusing when one is in flight.
- bool startBufferedTextSend(char* ownedBody, size_t bodyLen);
// Drain one chunk per client, finishing when every live client holds the whole frame. @xref{why-the-drain-runs-on-the-transport-tick}
void drainPreviewSend();
- // Same, for the in-flight full-state send to /ws clients. Called from tick20ms.
+ // Advance every open stream by one bounded chunk, generating each fragment as the previous one leaves. Called from tick20ms.
void drainStateSend();
+ // One client's stream, as far as `budget` bytes and the socket allow.
+ void drainStream(int client, size_t budget);
+ // A fragment found no memory: close the client, which reconnects and is sent its document again.
+ void refuseFragment(int client);
+ // Give each client due a document its stream, from the baseline taken this pass.
+ void startStreams();
+ // Generate a stream's next fragment and its frame header, false when even one fragment does not fit in memory.
+ bool nextStreamFrame(int client);
+ // Free a stream's fragment and close it.
+ void endStream(StateStream& s);
+ // Whether any client is mid-document, which holds every other frame on the control channel.
+ bool anyStreamActive() const;
+ // Whether some client alone waits for a full state.
+ bool anyClientResync() const;
+ // Store a client's subscription, resyncing when it changed.
+ void applySubscription(int client, const char* body);
+ // The module at a cursor's path, or null when the deepest index has run past its siblings.
+ MoonModule* moduleAt(const TreeCursor& c) const;
// Derived from free contiguous memory, so a tight board takes small bites and a roomy one drains fast.
size_t drainChunkBytes() const;
@@ -454,12 +540,20 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster {
ScratchBuffer leafHashes_{*this};
uint16_t leafHashCount_ = 0;
bool fullResyncPending_ = true; // send a full state next push (set on connect / structural change)
- // Emit only the leaves whose value hash moved, returning how many there were. @xref{state-push-a-diff-on-the-wire} @xref{what-the-diff-buys}
+ // Emit only the leaves whose value hash moved, returning how many there were. @xref{state-push-a-diff-on-the-wire}
uint16_t buildStatePatch(JsonSink& sink);
- // The same, over the leaves `walk` visits, which is the whole tree or the modules that asked early.
- template uint16_t buildPatch(JsonSink& sink, Walk&& walk);
- // Send one patch frame to every control-channel client.
- void sendPatch(const JsonSink& sink);
+ // One module's changed leaves, a slice of the patch entries that each client takes when its subscription covers the module.
+ struct PatchSlice { const MoonModule* mod; uint32_t begin; uint32_t end; uint16_t count; };
+ // Sized at each baseline to the modules it covers, one slice each.
+ ScratchBuffer patchSlices_{*this};
+ // The changed leaves over what `walk` visits, as comma-led entries in one slice per module, returning how many there were.
+ template uint16_t collectPatch(JsonSink& entries, uint16_t& sliceCount, Walk&& walk);
+ // Wrap the slices `sub` covers as one patch frame, null covering all, returning how many entries it holds.
+ uint16_t writeClientPatch(JsonSink& out, const JsonSink& entries, uint16_t sliceCount, const Subscription* sub);
+ // Collect over `walk`, then send each control client the share its subscription covers; a client about to get a full state is skipped when `skipResyncing`.
+ template void sendPatches(Walk&& walk, bool skipResyncing = false);
+ // Whether some client's subscription covers `mod`, so its leaves are worth hashing.
+ bool anyClientCovers(const MoonModule* mod) const;
// Modules whose changed values go out ahead of the periodic patch, by name since one may be deleted before the tick.
static constexpr uint8_t kSoonModules = 4;
@@ -468,17 +562,22 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster {
char soon_[kSoonModules][16] = {}; // a module name's own size
uint8_t soonCount_ = 0;
uint32_t soonSentMs_ = 0;
- // Patch the modules that asked, and forget them.
+ // Patch the modules that asked over the whole tree, and forget them, for a test.
uint16_t buildSoonPatch(JsonSink& sink);
+ // Visit the leaves of the modules that asked, and forget them.
+ template void forEachSoonLeaf(Fn&& fn, bool everything = false);
// Send the early patch, held while a full state drains and paced by kSoonMs.
void pushSoonPatch();
// Routes a module's request to the live instance.
static void onValuesChanged(MoonModule* mod);
- // Re-hash every leaf without emitting, so the next patch reports only changes since now.
- void baselineLeafHashes();
- // Visit every UI leaf in the state's own order, templated so the lambda inlines.
- template void forEachStateLeaf(Fn&& fn);
- template void visitModuleLeaves(MoonModule* mod, Fn&& fn);
+ // Re-hash every subscribed leaf without emitting, so the next patch reports only changes since now; `everything` covers the whole tree, for a test.
+ void baselineLeafHashes(bool everything = false);
+ // Visit every UI leaf some client subscribes to, in the state's own order, templated so the lambda inlines; `everything` visits the whole tree, for a test.
+ template void forEachStateLeaf(Fn&& fn, bool everything = false);
+ // One module's own leaves, `fn` taking the module first so the patch knows whose slice each belongs to.
+ template void visitOwnLeaves(MoonModule* mod, Fn&& fn);
+ // A subtree's leaves, skipping a module no client covers unless `everything`.
+ template void visitModuleLeaves(MoonModule* mod, Fn&& fn, bool everything = false);
LeafHash* findLeaf(uint32_t pathHash);
// The next push must be a full state and a fresh baseline. @xref{when-a-patch-cannot-describe-the-change}
void requestFullResync() { fullResyncPending_ = true; }
@@ -531,7 +630,9 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster {
void serveState(platform::TcpConnection& conn);
void buildStateJson(JsonSink& sink);
void writeModuleJson(JsonSink& sink, MoonModule* mod);
- void writeControls(JsonSink& sink, MoonModule* mod);
+ // A module's fields before its controls, which an outline carries alone.
+ void writeModuleHeader(JsonSink& sink, MoonModule* mod);
+ static void writeControl(JsonSink& sink, const ControlDescriptor& c);
// Emit a module's status and severity where set, shared so the two endpoints stay in sync.
static void writeStatus(JsonSink& sink, MoonModule* mod);
@@ -552,7 +653,12 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster {
void serveWledDeviceJson(platform::TcpConnection& conn); ///< /json, the surface Home Assistant's WLED integration reads
void serveWledPresets(platform::TcpConnection& conn); ///< /presets.json, the looks Home Assistant lists
void handleWledState(platform::TcpConnection& conn, const char* body);
- void pollWledStateFromWebSockets(); ///< read app's slider/toggle sent over /ws
+ /// Read every client's uplink: preview requests on `/wsp`, and on `/ws` control writes, subscriptions, resync requests, Ping, Close and the WLED app's frame.
+ void pollUplinks();
+ void pollPreviewClient(int client);
+ void pollControlClient(int client);
+ // What a text frame from a control client asks for.
+ void onTextFrame(int client, const char* body);
void writeWledInfoBody(JsonSink& sink, const char* name, const uint8_t mac[6]);
void writeWledName(JsonSink& sink, const char* name); // 💫-prefixed WLED name (HA marker)
void writeWledStateBody(JsonSink& sink);
diff --git a/src/core/system/ImprovProvisioningModule.h b/src/core/system/ImprovProvisioningModule.h
index 71fb44577..8b5396f7b 100644
--- a/src/core/system/ImprovProvisioningModule.h
+++ b/src/core/system/ImprovProvisioningModule.h
@@ -26,7 +26,7 @@ namespace mm {
///
/// ## Transports
///
-/// The listener serves an external USB-to-UART bridge and a native USB port alike.
+/// The listener serves an external USB-to-UART bridge and a native USB port alike; on a Teensy it is the USB serial port, polled from this module's tick.
/// Without either, the access-point flow remains, the device serving its own network.
///
/// ## The commands
@@ -105,6 +105,7 @@ class ImprovProvisioningModule : public MoonModule {
/// Apply any pending operation, polled every tick so a burst of them installs briskly.
void tick() MM_NONBLOCKING override {
+ if constexpr (platform::hasImprov) platform::improvPoll();
if (pendingOpReady_.load(std::memory_order_acquire) && scheduler_) {
// The frame was acknowledged on receipt, so a failure must surface here instead.
const StateDocumentResult r = applyStateDocument(*scheduler_, pendingOp_);
diff --git a/src/core/system/MoonCloudModule.h b/src/core/system/MoonCloudModule.h
index 3801d5540..e9aba223e 100644
--- a/src/core/system/MoonCloudModule.h
+++ b/src/core/system/MoonCloudModule.h
@@ -5,6 +5,7 @@
#include
#include
#include
+#include
#include
#include "core/module/MoonModule.h"
@@ -15,29 +16,97 @@ namespace mm {
/// The container for everything MoonLight does with a server we run.
///
-/// It holds no controls and does no work of its own.
-/// Each thing MoonCloud does is a child with its own consent: Stats, Talk, and Sync to come.
-/// A user who wants a joint lightshow has not agreed to usage reporting.
+/// It does no work of its own. Each thing MoonCloud does is a child with its own consent: Stats, Talk, and Sync to come. A user who wants a joint lightshow has not agreed to usage reporting.
+///
+/// @moreinfo
+///
+/// ## The browser relay, on a board without HTTPS
+///
+/// A board with no TLS cannot post, so its one outbound call goes into a one-slot outbox instead, a hidden control the open page reads.
+/// The page posts it to the server, which accepts a cross-origin request, and writes the request's `id` to the hidden `outbox sent` to empty it. A report then goes out whenever someone has the page open, enough for a usage count.
+/// Two pages open at once can both post the waiting request before either empties it, a rare double the count tolerates. Neither control exists on a board with HTTPS, so the relay costs it nothing but the check.
class MoonCloudModule : public MoonModule {
public:
+ /// A board without HTTPS gets its outbox at construction, so the relay's controls are declared with the rest.
+ explicit MoonCloudModule(bool relay = !platform::httpsAvailable())
+ : outbox_(relay ? static_cast(platform::alloc(kOutboxBytes)) : nullptr) {
+ if (!outbox_) return;
+ outbox_[0] = '\0';
+ setDynamicBytes(kOutboxBytes); // the card's memory readout counts it
+ }
+ /// Free the outbox, where there is one.
+ ~MoonCloudModule() override { platform::free(outbox_); }
+
/// Always runs: the children carry the real consent, so disabling the container is ambiguous.
bool respectsEnabled() const MM_NONBLOCKING override { return false; }
/// How long an outbound call may take before it is abandoned.
static constexpr uint32_t kTimeoutMs = 4000;
- /// The one outbound call, over HTTPS alone since the address is a compiled-in constant.
- bool post(const char* path, const char* body) const {
+ /// The relay's two hidden controls where there is an outbox, then the children's.
+ void defineControls() override {
+ controls_.clear();
+ if (outbox_) {
+ // Read-only, which also keeps it out of the saved settings: a request is transient.
+ controls_.addText("outbox", outbox_, kOutboxBytes);
+ controls_.setReadOnly(controls_.count() - 1, true);
+ controls_.setHidden(controls_.count() - 1, true);
+ controls_.addControl("outbox sent", acked_, 0, INT32_MAX);
+ controls_.setHidden(controls_.count() - 1, true);
+ }
+ MoonModule::defineControls();
+ }
+
+ // A restored settings file carries the last acknowledged id, which a new request must never match.
+ /// Number the next request after the last one acknowledged, which the settings restored.
+ void setup() override {
+ lastId_ = acked_;
+ MoonModule::setup();
+ }
+
+ // Only for the waiting request's id, so a late acknowledgement of an earlier one leaves a newer one waiting.
+ /// Empty the outbox once the page has delivered the request waiting in it.
+ void onControlChanged(const char* name) override {
+ if (outbox_ && outbox_[0] && name && std::strcmp(name, "outbox sent") == 0 && acked_ == pendingId_) {
+ outbox_[0] = 0;
+ notifyValuesChanged();
+ }
+ }
+
+ /// Whether a post can leave this board at all: over HTTPS, or through the page.
+ bool canSend() const { return platform::httpsAvailable() || outbox_ != nullptr; }
+ /// Whether a post waits for the page rather than leaving at once.
+ bool relaysThroughBrowser() const { return outbox_ != nullptr; }
+
+ /// Whether a request still waits for the page to deliver it.
+ bool outboxWaiting() const { return outbox_ && outbox_[0]; }
+
+ /// The one outbound call, over HTTPS or into the outbox for the page (@xref{the-browser-relay-on-a-board-without-https}); false when it could not be handed off.
+ bool post(const char* path, const char* body) {
+ if (outbox_) {
+ if (outbox_[0]) return false;
+ pendingId_ = lastId_ < INT32_MAX ? lastId_ + 1 : 1;
+ lastId_ = pendingId_;
+ const int n = std::snprintf(outbox_, kOutboxBytes, "{\"id\":%ld,\"path\":\"%s\",\"body\":%s}",
+ static_cast(pendingId_), path, body);
+ if (n < 0 || static_cast(n) >= kOutboxBytes) { outbox_[0] = 0; return false; }
+ notifyValuesChanged(); // to the page on the next transport tick
+ return true;
+ }
char url[192];
std::snprintf(url, sizeof(url), "https://%s%s", kHost, path);
return platform::httpsPost(url, body, kTimeoutMs);
}
-
-
private:
// Compiled in rather than configurable; this name because a custom domain's CA is absent.
static constexpr const char* kHost = "mooncloud-stats.moonmodules.workers.dev";
+ // One request: a usage report is well under this, and a talk message is capped at 280 characters.
+ static constexpr uint16_t kOutboxBytes = 2048;
+ char* outbox_ = nullptr; ///< the request waiting for the page, on a board without HTTPS only
+ int32_t acked_ = 0; ///< the id the page last delivered, which it writes to `outbox sent`
+ int32_t pendingId_ = 0; ///< the id of the request waiting in the outbox
+ int32_t lastId_ = 0; ///< the id given last
};
diff --git a/src/core/system/MoonStatsModule.h b/src/core/system/MoonStatsModule.h
index 735c89a5d..d7f75d5e0 100644
--- a/src/core/system/MoonStatsModule.h
+++ b/src/core/system/MoonStatsModule.h
@@ -207,7 +207,13 @@ class MoonStatsModule : public MoonModule {
/// Say what this setting exchanges, on the status slot rather than in a control of its own.
void refreshStatus() {
- if (consent_) clearStatus();
+ // Where the report cannot leave at once, consent says how it does, or that it cannot, rather than letting the charts stay silent.
+ if (consent_ && cloudRelays())
+ setStatus("This board has no HTTPS client, so its open page sends the report for it.", Severity::Status);
+ else if (consent_ && !canSend())
+ setStatus("This board has no HTTPS client, so it cannot send its report: nothing from it reaches the charts.",
+ Severity::Warning);
+ else if (consent_) clearStatus();
else setStatus("Off. Switch on to share what hardware you run, once per install or upgrade "
"and whenever you press send update: "
"the empty charts below are exactly what it contributes to, and what you get "
@@ -224,7 +230,7 @@ class MoonStatsModule : public MoonModule {
// Every outcome says something, and consent is re-read since a write can arrive from the API.
if (!consent_) { setOwnStatus("Switch consent on first: nothing is sent while it is off.",
Severity::Warning); return; }
- if (!platform::httpsAvailable()) {
+ if (!canSend()) {
setOwnStatus("This build cannot send: it was compiled without an HTTPS client.",
Severity::Error);
return;
@@ -240,7 +246,8 @@ class MoonStatsModule : public MoonModule {
return;
}
if (sendReport(MoonStatsEvent::Refresh)) {
- setOwnStatus("Sent. The charts below now describe this device as it is now.",
+ setOwnStatus(cloudRelays() ? "Handed to this page, which sends it within a second."
+ : "Sent. The charts below now describe this device as it is now.",
Severity::Status);
} else {
setOwnStatus("Could not reach the server. Nothing was sent; press send update to try again.",
@@ -310,9 +317,18 @@ class MoonStatsModule : public MoonModule {
/// Send the automatic report once every guard below passes.
void tick1s() MM_NONBLOCKING override {
MoonModule::tick1s();
+ // A report handed to the page is marked once the outbox empties, and nothing else goes until then.
+ if (relayPending_) {
+ const auto* cloud = static_cast(parent());
+ if (!cloud || !cloud->outboxWaiting()) {
+ markReported();
+ relayPending_ = false;
+ }
+ return;
+ }
if (!reportDue()) return;
// Never sends, so there is nothing to hand off and nothing to mark.
- if (!platform::httpsAvailable()) return;
+ if (!canSend()) return;
if (!platform::networkReady()) return; // try again next second
// Our own AP means no route out, so a send would fail and mark itself reported.
if (inApMode()) return;
@@ -322,6 +338,14 @@ class MoonStatsModule : public MoonModule {
sendReport(dueEvent());
}
+ /// Whether a report can leave: over HTTPS, or through the page where the container relays.
+ bool canSend() const { return platform::httpsAvailable() || cloudRelays(); }
+ /// Whether the container hands the report to the open page rather than sending it.
+ bool cloudRelays() const {
+ const auto* cloud = static_cast(parent());
+ return cloud && cloud->relaysThroughBrowser();
+ }
+
/// Whether we serve our own access point, asked of the platform so no copy goes stale.
bool inApMode() const { return platform::wifiApConnected(); }
@@ -353,9 +377,14 @@ class MoonStatsModule : public MoonModule {
// Through the container, which owns the address. Whether it was accepted is reported.
bool sent = false;
- if (auto* cloud = static_cast(parent())) {
+ if (auto* cloud = static_cast(parent())) {
sent = cloud->post("/api/report", body.data());
}
+ // Through the page, the report counts once the page has delivered it, so a reboot before then sends it again.
+ if (cloudRelays()) {
+ relayPending_ = sent;
+ return sent;
+ }
// The automatic report marks itself either way; a failed press must not.
if (kind != MoonStatsEvent::Refresh || sent) markReported();
return sent;
@@ -383,6 +412,7 @@ class MoonStatsModule : public MoonModule {
private:
bool consent_ = false; ///< whether the user has agreed to report
+ bool relayPending_ = false; ///< a report waits in the container's outbox for the page
char reportedVersion_[32] = {}; ///< the version that last reported
char runningVersion_[32] = {}; ///< the version running now
};
diff --git a/src/core/system/MoonTalkModule.h b/src/core/system/MoonTalkModule.h
index f7ef2e965..d32cfa105 100644
--- a/src/core/system/MoonTalkModule.h
+++ b/src/core/system/MoonTalkModule.h
@@ -120,7 +120,7 @@ class MoonTalkModule : public MoonModule {
body.flush();
// Through the container, which owns the address.
- auto* cloud = static_cast(parent());
+ auto* cloud = static_cast(parent());
return cloud && cloud->post("/api/talk", body.data());
}
diff --git a/src/core/system/SystemModule.h b/src/core/system/SystemModule.h
index 56b55ba43..f1970eaba 100644
--- a/src/core/system/SystemModule.h
+++ b/src/core/system/SystemModule.h
@@ -127,14 +127,21 @@ class SystemModule : public MoonModule {
if (totalHeapVal_ > totalInternalVal_) {
controls_.addProgress("psram", psramUsedVal_, totalHeapVal_ - totalInternalVal_);
// A compile-time string, so it binds read-only and adds nothing on a board without it.
- if (const char* pt = platform::psramType(); pt && pt[0])
+ if (const char* pt = platform::psramType(); pt && pt[0]) {
controls_.addReadOnly("psramType", const_cast(pt));
+ controls_.setAdvanced(controls_.count() - 1); // for whoever asks why a board is slow, not for running a show
+ }
}
controls_.addReadOnly("maxBlock", maxBlockStr_, sizeof(maxBlockStr_));
// Firmware identity lives on the firmware card, and filesystem usage on the file manager.
if (chipFlashVal_ > 0) {
controls_.addReadOnly("flash", flashStr_, sizeof(flashStr_));
+ // Read once from the hardware: a board updated over the network keeps its bootloader's mode, and DIO fetches code over two lines where QIO uses four.
+ if (const char* fm = platform::flashMode(); fm && fm[0]) {
+ controls_.addReadOnly("flashMode", const_cast(fm));
+ controls_.setAdvanced(controls_.count() - 1); // expert, as psramType
+ }
}
// These never change at runtime, so each binds straight to the platform's own buffer.
@@ -211,11 +218,11 @@ class SystemModule : public MoonModule {
/// Delete every setting and preset, so the next boot starts as freshly installed; scripts and other files stay.
static bool factoryReset() { return FilesystemModule::removeTree(FilesystemModule::CONFIG_DIR); }
- /// Accept printable ASCII within the buffer, which every write path checks.
+ /// Accept printable ASCII within the buffer, which every write path checks; empty is the unset model a board starts with.
static bool validateDeviceModel(const char* value) {
if (!value) return false;
size_t n = std::strlen(value);
- if (n == 0 || n >= 32) return false; // 1..31 (32-byte buffer, NUL-terminated)
+ if (n >= 32) return false; // 0..31 (32-byte buffer, NUL-terminated)
for (size_t i = 0; i < n; i++) {
unsigned char b = static_cast(value[i]);
if (b < 0x20 || b > 0x7E) return false;
diff --git a/src/core/util/ImprovOpReassembler.h b/src/core/util/ImprovOpReassembler.h
index f8e55e6b7..350dc145c 100644
--- a/src/core/util/ImprovOpReassembler.h
+++ b/src/core/util/ImprovOpReassembler.h
@@ -1,5 +1,6 @@
#pragma once
+#include
#include
#include
#include
@@ -13,15 +14,13 @@ namespace mm {
/// ## Where it sits
///
/// This is the pure logic behind the device's 0xFC handler, which treats Improv as REST over serial.
-/// The platform layer in `platform_esp32_improv.cpp` owns the serial I/O, reading frames, sending acks and errors, and holding the single-buffer `opReady` atomic; it hands each chunk's `[seq][last][bytes]` here.
+/// Each platform's listener owns its transport and hands each APPLY_OP payload to @ref improvApplyOp, which feeds the chunks here and holds the `opReady` handshake.
/// That is the same core-against-platform line `ImprovFrame.h` draws, so the reassembly and its sequence guard are proven on the desktop without hardware.
///
/// ## Why the sequence guard is real
///
-/// A chunk carries `[seq][last][chunk bytes]`.
-/// Sequence 0 starts a fresh op and resets the buffer, and every later chunk must be the next sequence in order.
-/// A duplicate, which an installer retry on a misread timeout produces, or an out-of-order chunk would splice garbage into the buffer.
-/// Both are rejected and the buffer is reset.
+/// A chunk carries `[seq][last][chunk bytes]`. Sequence 0 starts a fresh op and resets the buffer, and every later chunk must be the next sequence in order.
+/// A duplicate, which an installer retry on a misread timeout produces, or an out-of-order chunk would splice garbage into the buffer. Both are rejected and the buffer is reset.
/// USB serial delivers in order, but the installer's send is open-loop and can re-emit a chunk, so the guard guards something that happens.
///
/// Joins the chunks of one op into a NUL-terminated buffer the caller owns.
@@ -39,7 +38,7 @@ class ImprovOpReassembler {
/// Feed one chunk, `seq` being its 0-based index and `last` true on the final one.
Result feed(uint8_t seq, bool last, const uint8_t* chunk, size_t chunkLen) {
- // seq 0 always starts fresh, so a new op recovers after one that errored mid-stream.
+ // seq 0 always starts fresh, so a new op recovers after one that errored mid-stream: @xref{why-the-sequence-guard-is-real}.
if (seq == 0) {
len_ = 0;
nextSeq_ = 1;
@@ -86,5 +85,29 @@ class ImprovOpReassembler {
uint8_t nextSeq_ = 0; // next chunk index expected (0 = awaiting a fresh op)
};
+/// The vendor command carrying one state document over serial, and the error a refused chunk replies with.
+inline constexpr uint8_t kImprovCmdApplyOp = 0xFC;
+inline constexpr uint8_t kImprovErrorInvalidOp = 0x82;
+/// The protocol's own errors: a frame failing its checksum, and a command the listener does not take.
+inline constexpr uint8_t kImprovErrorInvalidRpc = 0x01;
+inline constexpr uint8_t kImprovErrorUnknownRpc = 0x02;
+
+/// What a transport replies to one APPLY_OP frame: an empty response for a taken chunk, the invalid-op error otherwise.
+enum class ImprovApplyOpReply : uint8_t { Ack, Refused };
+
+/// Handle one APPLY_OP payload, `[0xFC][seq][last][chunk…]`, for every transport (@xref{where-it-sits}): refused while a document waits or when malformed, and a last chunk sets `opReady`.
+inline ImprovApplyOpReply improvApplyOp(ImprovOpReassembler& reasm, const uint8_t* payload, size_t len, std::atomic& opReady) {
+ // Single-buffered: the acquire pairs with the consumer's release when it clears the flag after applying.
+ if (opReady.load(std::memory_order_acquire)) return ImprovApplyOpReply::Refused;
+ // `last` is a flag on the wire, so anything but 0 or 1 is a malformed frame rather than "more chunks".
+ if (len < 3 || payload[2] > 1) return ImprovApplyOpReply::Refused;
+ switch (reasm.feed(payload[1], payload[2] != 0, payload + 3, len - 3)) {
+ case ImprovOpReassembler::Result::Error: return ImprovApplyOpReply::Refused;
+ case ImprovOpReassembler::Result::Ready: opReady.store(true, std::memory_order_release); break;
+ case ImprovOpReassembler::Result::Continue: break;
+ }
+ return ImprovApplyOpReply::Ack;
+}
+
} // namespace mm
diff --git a/src/core/util/JsonSink.h b/src/core/util/JsonSink.h
index 37207fdc5..6d7fa7e4d 100644
--- a/src/core/util/JsonSink.h
+++ b/src/core/util/JsonSink.h
@@ -78,9 +78,13 @@ class JsonSink {
/// Append a string, growing or flushing as the mode requires.
void append(const char* s) {
+ if (s) append(s, std::strlen(s));
+ }
+ /// Append `len` bytes, a slice of a larger document that carries no terminator of its own.
+ void append(const char* s, size_t len) {
if (!s) return;
- if (conn_ && !stage_) { conn_->write(reinterpret_cast(s), std::strlen(s)); return; }
- while (*s) {
+ if (conn_ && !stage_) { conn_->write(reinterpret_cast(s), len); return; }
+ for (const char* end = s + len; s < end; ) {
if (conn_) {
if (pos_ == STAGE_SIZE) flushStage();
stage_[pos_++] = *s++;
diff --git a/src/core/util/TeensyImage.h b/src/core/util/TeensyImage.h
new file mode 100644
index 000000000..2ddac6ce8
--- /dev/null
+++ b/src/core/util/TeensyImage.h
@@ -0,0 +1,69 @@
+#pragma once
+
+/// @defgroup TeensyImage Vetting a Teensy 4 image
+/// @{
+/// The checks an uploaded image passes before it may overwrite a Teensy's only program: a Teensy 4 image, a MoonLight one, and for the firmware the board runs.
+///
+/// @moreinfo
+///
+/// ## Pure, so a host test pins them
+///
+/// A Teensy holds one program, so a wrong image copied over it leaves the board off the network until a USB flash: these checks are all that stands between an upload and that.
+/// They read bytes alone, so they live here rather than in the platform file, and a host test drives them with synthetic and real images alike.
+///
+/// ## The caller passes the mark
+///
+/// The search takes its prefix from the caller's own mark, so the prefix occurs once in a firmware image and a literal here cannot be found in its place.
+
+#include
+#include
+#include 
-- `ethBoard`: the board's Ethernet wiring by name; `Custom` exposes every pin.
+- `ethBoard`: the Ethernet wiring by name; `Custom` shows every pin, a fixed interface none.
- `ipSettings`: DHCP or Static; Static reveals `ip`, `gateway`, `subnet` and `dns`.
Gateway and DNS are optional. A Static setting that cannot work, such as a gateway outside the subnet, is not used: the interface keeps DHCP and the card says why. Changing the addressing of the interface the page reaches the device through asks first, naming where the device will be.
@@ -240,7 +240,7 @@ Browse and manage the device filesystem: a folder tree with an inline text edito
Known renames from [MIGRATING.md](../../reference/MIGRATING.md) apply before upload, then a report lists what needs an eye. Every file applies as it lands, bar network settings and the web server's `port`, which the dialog names.
- `show hidden`: reveal dot-prefixed files and folders, such as `.config`.
-- `filesystem`: read-only usage bar (used / total bytes, from the platform).
+- `flash`: read-only usage bar (used / total bytes, from the platform).
- `lastSaved`: read-only; how long ago config was persisted (read from the Filesystem engine).
Detail: [technical](moxygen/FileManagerModule.md)
diff --git a/docs/moonmodules/core/ui.md b/docs/moonmodules/core/ui.md
index 322a44b60..49733c4cb 100644
--- a/docs/moonmodules/core/ui.md
+++ b/docs/moonmodules/core/ui.md
@@ -48,7 +48,7 @@ Fixed top, 44 px, left-to-right:
5. **System stats**, `uptime · NN K free` (uptime `Xd Yh Zm Ws`, free heap KB), read from the
SystemModule controls in the WS state push; no separate polling endpoint.
6. **WebSocket dot**, green = connected, gray = disconnected. The socket auto-reconnects with
- exponential backoff (500 ms → 5 s) on close, and a page reload covers the rare half-dead socket.
+ exponential backoff from 200 ms to a 5 s ceiling on close, and a page reload covers the rare half-dead socket.
7. **Reboot button (⏻)**, red border via `data-crashed="true"` when `SystemModule.bootReason` is
PANIC / INT_WDT / TASK_WDT / BROWNOUT. Press-twice to confirm: the first click arms it (solid red), a second sends `POST /api/reboot`. Disarms after 3 s or on pointer-leave, no `confirm()` popup.
8. **Theme toggle (☀/🌙)**, flips `[data-theme]` on ``; preference in `localStorage['mm_theme']`.
@@ -67,7 +67,7 @@ A left column listing the root modules, one entry per top-level MoonModule.
- **Footer** pinned to the bottom of the nav: social links (GitHub, Discord, Reddit, YouTube, inline
SVG) and a `©