chore(profiles): derive the discovery rows from the OWNd profile (prep for OWNd #142)

.github/release-notes/2.0.0b16.md

@@ -31,6 +31,7 @@ See the cover docs for the details and the known limits.
 - **Diagnostics**: the gateway profile block reports the OWNd profile's real `model_name`, `display_name`, `event_keepalive_interval` and `keepalive_summary` instead of `Generic` / 90 s for every gateway.
 - **Setup**: when `myhome.yaml` has gateway sections but none under this gateway's MAC, a warning names the entry MAC and the section keys that match no configured gateway (#106).
 - **Setup**: a gateway added manually is asked who it is once the password is accepted (`*#13**15##`, then `*#1013**1##` on the shared code 200, the MAC only from models known to answer `*#13**12##`); a typed placeholder MAC or model is corrected from the reply.
+- **Docs tooling**: the "Profile Subsystems (advertised)" column of the startup discovery table in `docs/getting-started/hardware-compatibility.md` is derived from the installed OWNd profile (`supports_who`) for every row, the same rule the startup gate applies, so an OWNd profile change (such as dropping WHO 16 from the F461 / MyHomeServer1) regenerates the docs at the pin bump (`sync_documentation.py --update`, together with the `pinned_whos` fallback the CI jobs without OWNd use).
 - **Expose Home Assistant to the bus**: a declarative `expose` engine projects HA entities and the clock onto free SCS addresses: virtual WHO 1 / WHO 2 actuators that a wall control can drive (mirroring HA state back to its LEDs), WHO 4 virtual temperature probes, WHO 18 virtual energy meters and the WHO 13 bus clock. Custom actions run as Home Assistant scripts (#750).
 - **Covers**: `class:` in `myhome.yaml` now reaches WHO 2 covers (a `curtain` is no longer shown as a shutter; #39); Venetian blind slat tilt as timed slat pulses (`slat_tilt`, `slat_time`) (#508); impulse garage and gate covers on WHO 1 relays with optional safety approval, door-entry strike locks and doorbell events (#667); native broadcast for general covers behind an interface (#679); a cover stop overtakes other queued frames (#694); cover kinematics split into pluggable strategies (#739).
 - **Lighting**: 2-digit Area 0 point addresses `01`–`09` (#713); `where: "0004"` in `myhome.yaml` is resolved to `04`, the spelling the gateways accept and report, with a one-time warning, and registry entries created under the 4-digit spelling are moved to it; the duplicate entity and device that bus discovery had created under `04` are removed, their area, labels, icon and custom name carried over to the configured ones where those have none, and a light that bus discovery had created at the address of a WHO 1 switch or impulse cover goes with its lock / unlock buttons; the `A` / `PL` state attributes of such a light, switch or cover read `0` / `4` instead of `00` / `04`, as on bus-discovered entities (#147; the `0809` of #135 is a point of area 8, not of area 0, and stays as typed); ON dimmers on a fresh install are probed for brightness support (#742); timed turn-on uses the OWNd builders (#659); lights the startup sweep already reported are not polled again (#701); area and group re-syncs only skip once every member has echoed, counting only the members' own echoes in the commanded state (#740, #749).

scripts/update_gateway_profiles.py

@@ -177,89 +177,101 @@
     },
 ]
 
-# Authoritative startup discovery capabilities calibrated against OWNd.profiles and firmware_oracle.json.
-# ``queried_whos`` is what the OWNd profile advertises (recomputed from the profile and checked by
-# verify_profiles_against_oracle); ``sweep_omits`` lists the general requests of those WHOs the
+# Startup discovery rows. With an importable OWNd the WHOs a row advertises are derived from
+# the installed profile (``profile.supports_who``) over STARTUP_DISCOVERY_WHOS, exactly as the
+# runtime gate in gateway.py decides what to send, so the table follows an OWNd profile change
+# at the pin bump. ``pinned_whos`` is the row the pinned OWNd (the ``OWNd==`` pin in
+# manifest.json) yields: it is used only when OWNd is not importable (the quality-scale,
+# test-coverage and docs CI jobs run the docs check without it), so the generated table is the
+# same with and without OWNd. At a pin bump that changes a profile, update ``pinned_whos``
+# together with ``sync_documentation.py --update``; ``tests/test_gateway_profiles.py`` pins the
+# fallback to the committed table. ``sweep_omits`` lists the general requests of those WHOs the
 # startup sweep leaves out anyway because the gateway firmware drops them before the bus
 # (GENERAL_ZONE_REQUEST_REFUSED / SOUND_STATUS_REQUEST_REFUSED in custom_components/myhome/gateway.py;
-# tests/test_startup_requests_per_firmware.py checks these lists against the sweep itself).
+# tests/test_startup_requests_per_firmware.py checks these lists against the sweep itself, and
+# verify_profiles_against_oracle checks them against ``pinned_whos``); a request of a WHO the row
+# does not advertise is not an omission (nothing is sent for that WHO at all) and is left out of
+# the generated row. ``notes`` describes the row; where a profile may list or drop WHO 16 between
+# OWNd releases, ``notes_without_who16`` is used instead when the row does not carry WHO 16.
 GENERAL_ZONE_STATUS = "*#4*0##"
 SOUND_STATUS = "*#16*0*5##"
 GATEWAY_DISCOVERY_METADATA: list[dict[str, object]] = [
     {
         "model": "**MH202**",
         "profile_name": "MH202",
-        "queried_whos": [1, 2, 4, 5, 16, 18],
+        "pinned_whos": [1, 2, 4, 5, 16, 18],
         "sweep_omits": [GENERAL_ZONE_STATUS, SOUND_STATUS],
         "notes": "Burglar Alarm (`*#5*0##`) and Energy (`*#18*51*...`) queries. WHO 4 and WHO 16 are advertised, but the `010024` firmware drops `*#4*0##` and `*#16*0*5##` (own-firmware-oracle), so the sweep leaves both out and each climate zone asks `*#4*<WHERE>*0##` itself.",
     },
     {
         "model": "**H4890 / AM4890**",
         "profile_name": "H4890",
-        "queried_whos": [1, 2, 4, 5, 16, 18],
+        "pinned_whos": [1, 2, 4, 5, 16, 18],
         "sweep_omits": [],
         "notes": "Supports Burglar Alarm, Sound (`*#16*0*5##`), and Energy meter 51 queries.",
     },
     {
         "model": "**F454**",
         "profile_name": "F454",
-        "queried_whos": [1, 2, 4, 16, 18],
+        "pinned_whos": [1, 2, 4, 16, 18],
         "sweep_omits": [GENERAL_ZONE_STATUS],
         "notes": "Full lighting, cover, sound (`bt_difson` firmware-verified), and energy meter 51 queries. The `020051` firmware drops `*#4*0##` (own-firmware-oracle), so each climate zone asks `*#4*<WHERE>*0##` itself. Alarm query omitted.",
     },
     {
         "model": "**F452 / F453 / F453AV**",
         "profile_name": "F453AV",
-        "queried_whos": [1, 2, 4, 16, 18],
+        "pinned_whos": [1, 2, 4, 16, 18],
         "sweep_omits": [],
         "notes": "Lighting, cover, climate, sound (`bt_difson` firmware-verified on F453/F453AV), and energy meter 51 queries. Alarm query omitted.",
     },
     {
         "model": "**F461**",
         "profile_name": "F461",
-        "queried_whos": [1, 2, 4, 16, 18],
+        "pinned_whos": [1, 2, 4, 16, 18],
         "sweep_omits": [GENERAL_ZONE_STATUS, SOUND_STATUS],
         "notes": "Lighting, cover, and energy meter 51 queries. The `020011` firmware drops `*#4*0##` and `*#16*0*5##` (own-firmware-oracle, confirmed live on 2.0.11), so the sweep leaves both out; zones are polled with `*#4*<WHERE>##`. Alarm query omitted.",
+        "notes_without_who16": "Lighting, cover, and energy meter 51 queries. Sound is WHO 22 only (no `bt_difson`; every WHO 16 frame refused, own-firmware-oracle `020011`, confirmed live on 2.0.11). The firmware also drops `*#4*0##`, so the sweep leaves it out; zones are polled with `*#4*<WHERE>##`. Alarm query omitted.",
     },
     {
         "model": "**F455**",
         "profile_name": "F455",
-        "queried_whos": [1, 2, 4, 18],
+        "pinned_whos": [1, 2, 4, 18],
         "sweep_omits": [],
         "notes": "Entry-level gateway: lighting, cover, climate, and energy meter 51 queries. Sound (WHO 16/22) and Alarm (WHO 5) omitted per installer manual.",
     },
     {
         "model": "**MyHomeServer1**",
         "profile_name": "MyHomeServer1",
-        "queried_whos": [1, 2, 4, 16, 18],
+        "pinned_whos": [1, 2, 4, 16, 18],
         "sweep_omits": [GENERAL_ZONE_STATUS, SOUND_STATUS],
         "notes": "Lighting, cover, and energy meter 51 queries. The firmware drops `*#4*0##` and `*#16*0*5##` (own-firmware-oracle `028206`, confirmed live on 3.87.13), so the sweep leaves both out; zones are polled with `*#4*<WHERE>##`. Alarm query omitted.",
+        "notes_without_who16": "Lighting, cover, and energy meter 51 queries. Sound is WHO 22 only (no `bt_difson`; every WHO 16 frame refused, own-firmware-oracle `028206`, confirmed live on 3.87.13). The firmware also drops `*#4*0##`, so the sweep leaves it out; zones are polled with `*#4*<WHERE>##`. Alarm query omitted.",
     },
     {
         "model": "**MH201**",
         "profile_name": "MH201",
-        "queried_whos": [1, 2, 4, 16, 18],
+        "pinned_whos": [1, 2, 4, 16, 18],
         "sweep_omits": [GENERAL_ZONE_STATUS],
         "notes": "Lighting, cover, climate, sound (unmeasured class default), and energy meter 51 queries. Not in own-firmware-oracle; a physical MH201 3.3.6 returned no zone frame to `*#4*0##` although its zones report (`tests/fixtures/traces/issue_466`), so the sweep leaves it out and each climate zone asks `*#4*<WHERE>*0##` itself. Alarm query omitted.",
     },
     {
         "model": "**MH200** *(Legacy)*",
         "profile_name": "MH200",
-        "queried_whos": [1, 2, 4, 16],
+        "pinned_whos": [1, 2, 4, 16],
         "sweep_omits": [GENERAL_ZONE_STATUS],
         "notes": "Lighting, cover, climate, and sound queries (`bt_difson` hardware and firmware-verified). The `020102` firmware drops `*#4*0##` (own-firmware-oracle), so its climate zones ask `*#4*<WHERE>*0##` themselves. Alarm and energy queries omitted.",
     },
     {
         "model": "**MH200N**",
         "profile_name": "MH200N",
-        "queried_whos": [1, 2, 4, 16],
+        "pinned_whos": [1, 2, 4, 16],
         "sweep_omits": [],
         "notes": "Lighting, cover, climate, and sound queries (`bt_difson` hardware and firmware-verified). The `010108` firmware forwards `*#4*0##` (own-firmware-oracle), unlike the MH200. Alarm and energy queries omitted.",
     },
     {
         "model": "**H4684 / H/L4684**",
         "profile_name": "H4684",
-        "queried_whos": [1, 2, 4, 16, 18],
+        "pinned_whos": [1, 2, 4, 16, 18],
         "sweep_omits": [GENERAL_ZONE_STATUS],
         "notes": "Lighting, cover, climate, sound, and energy meter 51 queries (`bt_multi` WHO 22 also enabled on firmware >= `6.0.0`). Both `020054` (own-firmware-oracle) and live `6.0.11` (`tests/fixtures/traces/issue_466`) drop `*#4*0##`, so each climate zone asks `*#4*<WHERE>*0##` itself; the SSDP spellings `H/L4684` and `HL4684` resolve to `H4684Profile`. Alarm query omitted.",
     },
@@ -283,6 +295,19 @@ def cast_list(value: object) -> list[object]:
     return list(value) if isinstance(value, (list, tuple)) else []
 
 
+def pinned_whos(row: dict[str, object]) -> list[int] | None:
+    """Return ``row["pinned_whos"]`` as WHO numbers, or ``None`` when it is not a list of them."""
+    pinned = row.get("pinned_whos")
+    if not isinstance(pinned, (list, tuple)) or not all(isinstance(w, int) and not isinstance(w, bool) for w in pinned):
+        return None
+    return list(pinned)
+
+
+def in_sweep_order(whos: list[int]) -> list[int]:
+    """Return ``whos`` in the order of STARTUP_DISCOVERY_WHOS, without repeats (what the sweep sends)."""
+    return [w for w in STARTUP_DISCOVERY_WHOS if w in whos]
+
+
 def extract_gateway_models_from_const(const_path: Path = CONST_PY) -> list[str]:
     """Parse SUPPORTED_GATEWAY_MODELS from custom_components/myhome/const.py using AST."""
     if not const_path.exists():
@@ -448,10 +473,28 @@ def build_options_block() -> str:
     return f"{OPTIONS_START_MARKER}\n{table}\n{OPTIONS_END_MARKER}"
 
 
+def discovery_whos_for(profile: object) -> list[int]:
+    """The startup discovery WHOs the runtime sends to a gateway with ``profile``.
+
+    Mirrors ``MyHOMEGatewayHandler._profile_supports_who``: a WHO is queried only when
+    the OWNd profile advertises it.
+    """
+    supports = getattr(profile, "supports_who", None)
+    if not callable(supports):
+        return []
+    return [w for w in STARTUP_DISCOVERY_WHOS if bool(supports(w))]
+
+
 def calibrate_discovery_with_ownd(
     discovery_metadata: list[dict[str, object]] | None = None,
 ) -> list[dict[str, object]]:
-    """Dynamically calibrate startup discovery WHOs against OWNd.profiles."""
+    """Derive the startup discovery rows from OWNd.profiles.
+
+    ``queried_whos`` (the advertised column), the ``sweep_omits`` that still apply and the
+    sound wording follow the installed profile. Without an importable OWNd the rows carry
+    ``pinned_whos`` (what the pinned OWNd yields), so the generated table is the same either
+    way as long as the pin and the fallback agree; nothing else is guessed.
+    """
     try:
         from OWNd.profiles import get_gateway_profile
     except ImportError:
@@ -460,15 +503,24 @@ def calibrate_discovery_with_ownd(
     calibrated = copy.deepcopy(
         GATEWAY_DISCOVERY_METADATA if discovery_metadata is None else discovery_metadata
     )
-    if get_gateway_profile is not None:
-        for row in calibrated:
+    for row in calibrated:
+        pinned = pinned_whos(row) or []
+        row.pop("pinned_whos", None)
+        without_who16 = row.pop("notes_without_who16", None)
+        if get_gateway_profile is None:
+            queried = in_sweep_order(pinned)
+        else:
             prof_name = row.get("profile_name")
-            if prof_name:
-                prof = get_gateway_profile(str(prof_name))
-                if prof is not None:
-                    row["queried_whos"] = [
-                        w for w in STARTUP_DISCOVERY_WHOS if prof.supports_who(w)
-                    ]
+            prof = get_gateway_profile(str(prof_name)) if prof_name else None
+            queried = discovery_whos_for(prof) if prof is not None else []
+        row["queried_whos"] = queried
+        # A general request of a WHO the row does not advertise is not an omission of the
+        # sweep (nothing is sent for that WHO at all), so it leaves the generated row.
+        row["sweep_omits"] = [
+            frame for frame in cast_list(row.get("sweep_omits")) if _frame_who(str(frame)) in queried
+        ]
+        if queried and without_who16 and 16 not in queried:
+            row["notes"] = without_who16
     return calibrated
 
 
@@ -482,8 +534,8 @@ def generate_gateway_discovery_table() -> str:
     for row in rows:
         model = row["model"]
         whos = row["queried_whos"]
-        who_str = "WHO " + ", ".join(str(w) for w in whos) if isinstance(whos, list) else str(whos)
-        sent = "WHO " + ", ".join(str(w) for w in sweep_whos(row))
+        who_str = "WHO " + ", ".join(str(w) for w in whos) if whos else "none"
+        sent = "WHO " + ", ".join(str(w) for w in sweep_whos(row)) if whos else "none"
         omitted = cast_list(row.get("sweep_omits"))
         if omitted:
             sent += " (firmware drops " + ", ".join(f"`{frame}`" for frame in omitted) + ")"
@@ -587,17 +639,42 @@ def _verdict(product: str, frame: str) -> tuple[str, str] | None:
     for row in GATEWAY_DISCOVERY_METADATA:
         prof_name = str(row.get("profile_name", ""))
         prof = get_gateway_profile(prof_name)
-        expected_whos = [w for w in STARTUP_DISCOVERY_WHOS if prof.supports_who(w)]
-        if row.get("queried_whos") != expected_whos:
+        if isinstance(prof, GenericGatewayProfile):
+            violations.append(
+                f"Discovery row {row.get('model')} names an OWNd profile that does not exist: "
+                f"{prof_name} (resolves to the generic profile)."
+            )
+        pinned = pinned_whos(row)
+        if pinned is None:
+            violations.append(
+                f"Discovery row {row.get('model')} pinned_whos={row.get('pinned_whos')!r} is not a list of WHO numbers."
+            )
+            pinned = []
+        if [w for w in pinned if w not in STARTUP_DISCOVERY_WHOS]:
+            violations.append(
+                f"Discovery row {row.get('model')} pinned_whos={pinned} names a WHO the startup sweep never asks for."
+            )
+        elif pinned != in_sweep_order(pinned):
+            violations.append(
+                f"Discovery row {row.get('model')} pinned_whos={pinned} is not in startup sweep order "
+                f"{in_sweep_order(pinned)} (a repeated or reordered WHO)."
+            )
+        # The pin may differ from the installed profile only by WHO 16, and only on a row that
+        # declares the WHO 16 switch (``notes_without_who16``): the one profile change the
+        # fallback absorbs between the pinned release and the next (OWNd #142). Any other
+        # difference is a stale or mistyped pin.
+        expected_whos = discovery_whos_for(prof)
+        tolerated = {16} if row.get("notes_without_who16") else set()
+        if (set(pinned) ^ set(expected_whos)) - tolerated:
             violations.append(
-                f"Discovery row {row.get('model')} queried_whos={row.get('queried_whos')} "
-                f"disagrees with {prof_name} profile {expected_whos}."
+                f"Discovery row {row.get('model')} pinned_whos={pinned} disagrees with the installed "
+                f"{prof_name} profile {expected_whos}."
             )
         for frame in cast_list(row.get("sweep_omits")):
-            if str(frame) not in (GENERAL_ZONE_STATUS, SOUND_STATUS) or _frame_who(str(frame)) not in expected_whos:
+            if str(frame) not in (GENERAL_ZONE_STATUS, SOUND_STATUS) or _frame_who(str(frame)) not in pinned:
                 violations.append(
                     f"Discovery row {row.get('model')} sweep_omits={frame!r} is not a general request "
-                    f"of an advertised WHO of the {prof_name} profile."
+                    f"of a WHO the row advertises (pinned_whos={pinned})."
                 )
 
     return violations