diff --git a/monitoring.yaml b/monitoring.yaml index 77fd2748..b5e40662 100644 --- a/monitoring.yaml +++ b/monitoring.yaml @@ -293,9 +293,11 @@ protocols: - name: "Low Liquidity" description: "V1 non-YV-collateral and all V2 vault liquidity <2%; combined V1/V2 YV-collateral unwind liquidity vs collateral at risk for Katana USDC, USDT, and WETH" - name: "V1 Governance" - description: "Pending supply caps, market removals, timelock changes, guardian changes" + description: "Pending supply caps, market removals (HIGH while the vault still supplies the market), timelock decreases, guardian changes" - name: "V2 Governance" - description: "Pending timelocked operations, owner/curator changes, sentinel/allocator/adapter changes" + description: "Pending timelocked operations (HIGH for removeAdapter of a funded adapter), owner/curator changes, sentinel/allocator/adapter changes" + - name: "V2 Adapter Share Burns" + description: "Pending MorphoMarketV1AdapterV2 burnShares (HIGH while the adapter holds the market) and burnShares timelock decreases, via eth_call" - name: "V2 Wrapped V1 Coverage" description: "V2 vault wrapping a v1 vault not in config.py:VAULTS_V1_BY_CHAIN" - name: "Safe Multisig" diff --git a/protocols/morpho/README.md b/protocols/morpho/README.md index ef40e5d5..65006ea9 100644 --- a/protocols/morpho/README.md +++ b/protocols/morpho/README.md @@ -8,16 +8,18 @@ Morpho governance monitoring is defined in the [Python script](./governance.py) The script checks if there are any new values pending in the timelock for a given vault. It detects the following changes: -- Changing timelock value, minimal values is 1 day. +- Decreasing the timelock (increases apply instantly, so only decreases are ever pending), minimal value is 1 day. The HIGH alert shows the current → new value and when it becomes executable. - Changing guardian address. - Changing supply caps, only to higher value than the current one, for both supply and withdraw markets. -- Removing of a market from the vault. +- Removing of a market from the vault (`submitMarketRemoval`). Once `removableAt` passes, `updateWithdrawQueue` can drop the market **even if the vault still supplies it**, writing that supply off. The monitor reads the vault's Morpho Blue position in the market: + - any non-zero supply → **HIGH** potential-loss alert with the amount at risk and its share of `totalAssets` (omitted if `totalAssets()` reverts, e.g. on a broken IRM — the check itself still runs), re-alerted once within 24h of `removableAt`. A removal first seen on an empty market escalates to this alert as soon as the vault supplies it; + - no supply → MEDIUM (routine cleanup). ### How to Add a New Vault Add a `VaultConfig` to `VAULTS_V1_BY_CHAIN` in [config.py](./config.py). Every configured V1 vault receives both market and governance monitoring. `governance.py` iterates every chain in that mapping, so a new chain needs no script change; a failure on one chain is logged and the remaining chains are still checked before the run fails. -Monitored chains: Mainnet, Base, Katana, HyperEVM, and Robinhood Chain (V2 only). HyperEVM uses `PROVIDER_URL_HYPEREVM` when set; if no `PROVIDER_URL_HYPEREVM*` variable exists it falls back to the public `https://rpc.hyperliquid.xyz/evm` (`PUBLIC_RPC_URLS` in [utils/chains.py](../../utils/chains.py)). RPC requests time out after 60 seconds so a stalled public endpoint cannot hang the daily profile. +Monitored chains: Mainnet, Base, Katana, HyperEVM, and Robinhood Chain (V2 only). HyperEVM uses `PROVIDER_URL_HYPEREVM` when set; if no `PROVIDER_URL_HYPEREVM*` variable exists it falls back to the public `https://rpc.hyperliquid.xyz/evm` (`PUBLIC_RPC_URLS` in [utils/chains.py](../../utils/chains.py)). Robinhood Chain likewise falls back to its official `https://rpc.mainnet.chain.robinhood.com`, used by the V2 adapter timelock reads. RPC requests time out after 60 seconds so a stalled public endpoint cannot hang the daily profile. ## Vaults & Markets @@ -154,8 +156,23 @@ Morpho's [Vault V2](https://github.com/morpho-org/vault-v2) replaces the v1 sing - [`governance_v2.py`](./governance_v2.py) — daily, pulls a per-vault governance **snapshot** from Morpho's GraphQL API (`vaultV2s.pendingConfigs` + `owner` / `curator` / `sentinels` / `allocators` / `adapters`) and diffs it against the persisted cache. Mirrors v1's pull-based approach (`pendingTimelock` / `pendingGuardian` / `pendingCap`) so RPC usage stays bounded. Alerts on: new pending timelocked operations, executed or revoked operations, owner / curator changes, sentinel / allocator / adapter set changes. - [`markets_v2.py`](./markets_v2.py) — hourly, GraphQL-only (no RPC): one `vaultV2s` query loads TVL, liquidity, and `MorphoMarketV1` adapter positions for every configured vault, then one `markets` query per chain loads state/bad debt. Applies the shared [risk.py](./risk.py) policy using each position's `supplyAssetsUsd`, and checks withdrawable liquidity (idle plus the vault's position in the liquidity adapter's market, capped by market cash) against the 2% V2 threshold for every vault, including YV-collateral ones, which `markets.py` also covers with the combined collateral-at-risk check. Non-`MorphoMarketV1` adapters fail the run (configured vaults are market-adapter only). +- [`adapter_timelocks_v2.py`](./adapter_timelocks_v2.py) — run by `governance_v2.py`, reads each `MorphoMarketV1AdapterV2`'s own timelock on-chain with batched `eth_call`s (no `eth_getLogs`), since the Morpho GraphQL API has no adapter-level timelock data. A pending `burnShares(marketId)` writes off the adapter's supply in that market (the V2 analogue of a V1 market removal): `executableAt(burnShares(marketId))` is read for every held market → HIGH with the amount at risk (stored shares at Morpho Blue market totals, never the IRM, which may revert); a burn on a market with no shares is LOW. A drop in `timelock(burnShares)` → MEDIUM. A failed RPC read still lets the GraphQL diff run, then fails the job. + - **TODO:** a burn submitted with non-canonical calldata (canonical + trailing bytes; Solidity ignores them, but `executableAt` is keyed by the exact bytes) is not detected. It needs the adapter's `Submit` events, which the Morpho API does not expose. - [`v2_decoders.py`](./v2_decoders.py) — selector→signature map and decoders for every v2 timelocked function (and the three `idData` tag prefixes used by `increaseAbsoluteCap`/`increaseRelativeCap`). Absolute caps are denominated in the *vault's* asset whatever id they are keyed by, so the decoder takes the vault's asset decimals/symbol; `type(uint128).max` renders as `unlimited`. +### Potential vault loss alerts + +Operations that can write off vault funds are **HIGH** for any non-zero amount, sent as their own message prefixed `🚨 POTENTIAL VAULT LOSS` (never bundled into the grouped diff), and re-alerted once when within 24 hours of execution. The loss alert is tracked separately from the "pending operation seen" cursor, so an operation first seen with nothing at risk escalates as soon as funds arrive instead of waiting for the reminder: + +| Vault | Operation | Condition | +|---|---|---| +| V1 | `submitMarketRemoval` | the vault still supplies the market | +| V2 | `removeAdapter(adapter)` | the adapter holds assets (`totalAssets` is the sum of adapter `realAssets`) | +| V2 | `decreaseTimelock(removeAdapter, …)` | any adapter holds assets | +| V2 adapter | `burnShares(marketId)` | the adapter holds assets in the market | + +Other pending V2 operations, including other timelock decreases and `abdicate`, stay MEDIUM. Pending V2 operations can be cancelled by the curator or a sentinel with `revoke(data)`; V1 removals by the curator or guardian with `revokePendingMarketRemoval`. + ### Grouped alerts Both governance monitors buffer their findings per vault in [`_alerts.py`](./_alerts.py) and send them as **one Telegram message per vault** — one header naming the vault and chain, one severity (the highest of the group), sections separated by `---`. A vault with several simultaneous changes no longer produces a burst of near-identical messages. Groups too long for a single message split into numbered `(i/N)` parts rather than being truncated, and the cache cursors that record "we alerted on this" are committed only after the send succeeds, so a Telegram failure is retried on the next run instead of being lost. diff --git a/protocols/morpho/_alerts.py b/protocols/morpho/_alerts.py index 2eb2e4d4..361375f8 100644 --- a/protocols/morpho/_alerts.py +++ b/protocols/morpho/_alerts.py @@ -16,6 +16,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from datetime import datetime from functools import partial from typing import Any, Callable, List @@ -34,12 +35,57 @@ _MESSAGE_OVERHEAD = 64 +# Header prefix for findings that can write off vault funds (market removal, +# adapter removal, share burn). They are sent as their own message so they never +# drown in a routine grouped diff. +LOSS_PREFIX = "🚨 POTENTIAL VAULT LOSS" + +# Pending loss operations are re-alerted once when their execution time is this close. +REMINDER_WINDOW_SECONDS = 24 * 60 * 60 + + +@dataclass(frozen=True) +class LossNotice: + """A potential-loss alert that is due for one pending operation.""" + + reminder: bool # render as the near-deadline reminder rather than the first alert + in_window: bool # the deadline is inside the reminder window, so mark the reminder as sent too + + +def due_loss_notice(deadline: int, loss_alerted_at: int, reminded_at: int) -> LossNotice | None: + """Decide whether a pending operation that puts funds at risk needs a loss alert now. + + The loss alert is tracked separately from the "pending operation seen" cursor, + so an operation first seen with nothing at risk (sent at a lower severity) still + escalates once funds arrive, instead of waiting for the reminder window. + + Args: + deadline: When the operation becomes executable; also the cache value. + loss_alerted_at: Cached deadline of the last loss alert sent for it. + reminded_at: Cached deadline of the last reminder sent for it. + + Returns: + The notice to send, or None if both the alert and the reminder already went out. + """ + in_window = int(datetime.now().timestamp()) >= deadline - REMINDER_WINDOW_SECONDS + if loss_alerted_at != deadline: + return LossNotice(reminder=False, in_window=in_window) + if in_window and reminded_at != deadline: + return LossNotice(reminder=True, in_window=True) + return None + + @dataclass class VaultAlert: - """One section of a vault's grouped Telegram message.""" + """One section of a vault's grouped Telegram message. + + ``standalone`` sections are sent as their own message with ``LOSS_PREFIX`` + instead of being grouped with the vault's other findings. + """ severity: AlertSeverity body: str + standalone: bool = False @dataclass @@ -53,6 +99,10 @@ def alert(self, severity: AlertSeverity, body: str) -> None: """Buffer one section of the vault's grouped message.""" self.alerts.append(VaultAlert(severity, body)) + def loss(self, body: str) -> None: + """Buffer a HIGH potential-loss finding, sent as its own message.""" + self.alerts.append(VaultAlert(AlertSeverity.HIGH, body, standalone=True)) + def defer(self, write: Callable[..., Any], *args: Any) -> None: """Buffer a cache write to apply once the alert is delivered.""" self.writes.append(partial(write, *args)) @@ -111,7 +161,18 @@ def send_vault_alerts(header: str, alerts: List[VaultAlert], protocol: str) -> N No-op when ``alerts`` is empty so callers don't have to guard. Every part carries the same header and the highest severity of the whole group, so a LOW section bundled with an owner change still pings the channel. + + Standalone (potential-loss) sections go out first, one message each, under a + ``LOSS_PREFIX`` header; the remaining sections are grouped as usual. """ + for alert in alerts: + if alert.standalone: + _send_group(f"{LOSS_PREFIX} — {header}", [alert], protocol) + _send_group(header, [a for a in alerts if not a.standalone], protocol) + + +def _send_group(header: str, alerts: List[VaultAlert], protocol: str) -> None: + """Send sections under one header at their highest severity, split into parts if long.""" if not alerts: return severity = max((a.severity for a in alerts), key=_SEVERITY_ORDER.index) diff --git a/protocols/morpho/_shared.py b/protocols/morpho/_shared.py index 9f22833d..6384c4d8 100644 --- a/protocols/morpho/_shared.py +++ b/protocols/morpho/_shared.py @@ -2,12 +2,14 @@ import time from dataclasses import dataclass +from datetime import datetime from typing import Any, Dict, Iterable, List, Optional import requests from utils.chains import Chain from utils.config import Config +from utils.formatting import format_duration, format_token_amount, format_with_suffix from utils.http_client import request_with_retry from utils.logger import get_logger @@ -81,6 +83,67 @@ def execute_graphql( raise error_type(f"Morpho GraphQL failed after {retries + 1} attempts for {context}") +MORPHO_BLUE_ABI_PATH = "protocols/morpho/abi/morpho_blue.json" + +# Morpho Blue SharesMathLib virtual offsets, used to convert supply shares to assets. +VIRTUAL_SHARES = 10**6 +VIRTUAL_ASSETS = 1 + + +def to_assets_down(shares: int, total_assets: int, total_shares: int) -> int: + """Convert Morpho Blue supply shares to assets, rounding down like ``SharesMathLib``. + + Uses the market's stored totals rather than an IRM-accrued view, so it can't + revert on a market whose IRM reverts; it only omits interest since the last update. + """ + return shares * (total_assets + VIRTUAL_ASSETS) // (total_shares + VIRTUAL_SHARES) + + +def format_ts(ts: int) -> str: + """Render a unix timestamp as local ``YYYY-MM-DD HH:MM:SS``.""" + return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") + + +def format_countdown(ts: int) -> str: + """Human-friendly time remaining until ``ts``, e.g. ``(3 days)``.""" + seconds = ts - int(datetime.now().timestamp()) + if seconds <= 0: + return "(now)" + days = seconds / 86400 + if days >= 1: + n = round(days) + return f"({n} day{'s' if n != 1 else ''})" + hours = seconds / 3600 + if hours >= 1: + n = round(hours) + return f"({n} hour{'s' if n != 1 else ''})" + n = max(round(seconds / 60), 1) + return f"({n} minute{'s' if n != 1 else ''})" + + +def format_timelock(seconds: int) -> str: + """Render a timelock duration; a zero timelock reads ``0s`` rather than ``now``.""" + return format_duration(seconds) if seconds > 0 else "0s" + + +def format_amount_at_risk(raw: int, decimals: int | None, symbol: str | None, total_assets: int | None) -> str: + """Render an at-risk amount with its share of the vault, e.g. ``1.20M USDC (12.50% of vault)``. + + Dust that would round to ``0.00`` is shown in raw units instead, because any + non-zero amount at risk is worth seeing exactly. + """ + if decimals is None: + amount = f"{raw:,} (raw units)" + else: + human = format_token_amount(raw, decimals) + amount = str(format_with_suffix(human)) if human >= 0.01 else f"{raw:,} raw units" + if symbol: + amount = f"{amount} {symbol}" + if total_assets: + amount = f"{amount} ({raw / total_assets:.2%} of vault)" + return amount + + def require_configured_keys( expected: Iterable[str], found: Iterable[str], diff --git a/protocols/morpho/abi/morpho_blue.json b/protocols/morpho/abi/morpho_blue.json new file mode 100644 index 00000000..fdd2731e --- /dev/null +++ b/protocols/morpho/abi/morpho_blue.json @@ -0,0 +1,30 @@ +[ + { + "inputs": [ + { "internalType": "Id", "name": "", "type": "bytes32" }, + { "internalType": "address", "name": "", "type": "address" } + ], + "name": "position", + "outputs": [ + { "internalType": "uint256", "name": "supplyShares", "type": "uint256" }, + { "internalType": "uint128", "name": "borrowShares", "type": "uint128" }, + { "internalType": "uint128", "name": "collateral", "type": "uint128" } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [{ "internalType": "Id", "name": "", "type": "bytes32" }], + "name": "market", + "outputs": [ + { "internalType": "uint128", "name": "totalSupplyAssets", "type": "uint128" }, + { "internalType": "uint128", "name": "totalSupplyShares", "type": "uint128" }, + { "internalType": "uint128", "name": "totalBorrowAssets", "type": "uint128" }, + { "internalType": "uint128", "name": "totalBorrowShares", "type": "uint128" }, + { "internalType": "uint128", "name": "lastUpdate", "type": "uint128" }, + { "internalType": "uint128", "name": "fee", "type": "uint128" } + ], + "stateMutability": "view", + "type": "function" + } +] diff --git a/protocols/morpho/adapter_timelocks_v2.py b/protocols/morpho/adapter_timelocks_v2.py new file mode 100644 index 00000000..0d25f3c0 --- /dev/null +++ b/protocols/morpho/adapter_timelocks_v2.py @@ -0,0 +1,283 @@ +"""On-chain checks of ``MorphoMarketV1AdapterV2``'s own timelock. + +The adapter's ``burnShares(marketId)`` is the V2 equivalent of a V1 +``submitMarketRemoval``: once executed it zeroes the adapter's supply shares in +that market, so ``realAssets`` drops and the vault takes the loss. It is +timelocked on the adapter itself, which the Morpho GraphQL API does not expose. + +Only batched ``eth_call`` reads are used (never ``eth_getLogs``): + +* **Pending burns** — ``executableAt(abi.encodeCall(burnShares, (id)))`` for every + held market. +* ``timelock(burnShares)`` — a shortened reaction window, after the fact. + +TODO: a burn submitted with non-canonical calldata (canonical + trailing bytes) +is not detected. ``executableAt`` is keyed by the exact calldata and Solidity +ignores trailing bytes, so such a burn executes under a key we can't derive. +Finding it needs the adapter's ``Submit`` events, which the Morpho API does not +expose (no adapter-level timelock data at all); add it once an indexer provides them. + +Amounts use the adapter's stored shares and Morpho Blue's stored market totals, +never ``expectedSupplyAssets``: that goes through the market's IRM, which may +revert, and burning shares of such a market is explicitly supported. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, List + +from web3 import Web3 + +from protocols.morpho._alerts import VaultDiff, due_loss_notice +from protocols.morpho._shared import ( + MORPHO_BLUE_ABI_PATH, + fetch_market_metadata, + format_amount_at_risk, + format_countdown, + format_timelock, + format_ts, + get_market_url, + to_assets_down, +) +from protocols.morpho.v2_decoders import selector_of +from utils.abi import load_abi +from utils.alert import AlertSeverity +from utils.cache import get_last_value_for_key_from_file, morpho_filename, morpho_key, write_last_value_to_file +from utils.chains import Chain +from utils.logger import get_logger +from utils.web3_wrapper import ChainManager + +logger = get_logger("morpho.adapter_timelocks_v2") + +ABI_MARKET_V1_ADAPTER_V2 = load_abi("protocols/morpho/abi/morpho_market_v1_adapter_v2.json") +ABI_MORPHO_BLUE = load_abi(MORPHO_BLUE_ABI_PATH) +BURN_SHARES_SELECTOR = selector_of("burnShares(bytes32)") + +# Cache value-type tags used with utils.cache.morpho_key. +BURN_TIMELOCK_TYPE = "v2_adapter_burn_timelock" +PENDING_BURN_TYPE = "v2_adapter_pending_burn" +PENDING_BURN_LOSS_TYPE = "v2_adapter_pending_burn_loss" +PENDING_BURN_REMINDER_TYPE = "v2_adapter_pending_burn_reminder" +PENDING_BURN_INDEX_TYPE = "v2_adapter_pending_burn_index" + + +@dataclass(frozen=True) +class PendingBurn: + """A submitted, not yet executed canonical ``burnShares`` for one market.""" + + market_id: str # 0x-prefixed hex + executable_at: int + shares: int # the adapter's stored supply shares in the market + assets: int # those shares valued at Morpho Blue's stored market totals + + +@dataclass(frozen=True) +class AdapterTimelockState: + """On-chain timelock state of one ``MorphoMarketV1AdapterV2``.""" + + adapter: str # checksummed + burn_timelock: int + pending_burns: List[PendingBurn] = field(default_factory=list) + + +def burn_shares_calldata(market_id: bytes) -> bytes: + """Canonical calldata of ``burnShares(marketId)``, the key of the adapter's ``executableAt`` mapping.""" + return BURN_SHARES_SELECTOR + market_id + + +def _execute(client: Any, functions: List[Any], block_number: int) -> List[Any]: + """Call contract functions as one JSON-RPC batch and check every call answered. + + ``functions`` are bound but not yet called (``contract.functions.x(args)``): + ``.call()`` must run inside the batch context, or it executes eagerly. + """ + if not functions: + return [] + with client.batch_requests() as batch: + for function in functions: + batch.add(function.call(block_identifier=block_number)) + responses = client.execute_batch(batch) + if len(responses) != len(functions): + raise ValueError(f"Expected {len(functions)} batch responses, got {len(responses)}") + return list(responses) + + +# ---------------------------------------------------------------------------- +# Cache helpers +# ---------------------------------------------------------------------------- + + +def _read_str(key: str) -> str: + raw = get_last_value_for_key_from_file(morpho_filename, key) + return str(raw) if raw else "" + + +def _read_int(key: str) -> int: + try: + return int(get_last_value_for_key_from_file(morpho_filename, key)) + except TypeError, ValueError: + return 0 + + +def _write(key: str, value: Any) -> None: + write_last_value_to_file(morpho_filename, key, value) + + +def _key(adapter: str, item: str, value_type: str) -> str: + return str(morpho_key(adapter.lower(), item, value_type)) + + +# ---------------------------------------------------------------------------- +# Loading +# ---------------------------------------------------------------------------- + + +def load_adapter_state(client: Any, adapter_address: str, block_number: int) -> AdapterTimelockState: + """Read the burn timelock and canonical pending burns of one adapter.""" + adapter = client.eth.contract(address=Web3.to_checksum_address(adapter_address), abi=ABI_MARKET_V1_ADAPTER_V2) + fns = adapter.functions + length, burn_timelock = _execute(client, [fns.marketIdsLength(), fns.timelock(BURN_SHARES_SELECTOR)], block_number) + markets: List[bytes] = _execute(client, [fns.marketIds(i) for i in range(length)], block_number) + executable_at = _execute(client, [fns.executableAt(burn_shares_calldata(m)) for m in markets], block_number) + burned = [(m, at) for m, at in zip(markets, executable_at) if at > 0] + if not burned: + return AdapterTimelockState(adapter=adapter.address, burn_timelock=burn_timelock) + + # Valuation only for the (rare) markets with a pending burn. + blue = client.eth.contract(address=_execute(client, [fns.morpho()], block_number)[0], abi=ABI_MORPHO_BLUE) + values = _execute( + client, [f for m, _ in burned for f in (fns.supplyShares(m), blue.functions.market(m))], block_number + ) + pending = [] + for index, (market_id, at) in enumerate(burned): + shares, market = values[index * 2], values[index * 2 + 1] + pending.append(PendingBurn(Web3.to_hex(market_id), at, shares, to_assets_down(shares, market[0], market[1]))) + return AdapterTimelockState(adapter=adapter.address, burn_timelock=burn_timelock, pending_burns=pending) + + +def load_adapter_states(chain: Chain, adapters: List[str]) -> List[AdapterTimelockState]: + """Load the timelock state of every ``MorphoMarketV1AdapterV2`` of one vault.""" + if not adapters: + return [] + client = ChainManager.get_client(chain) + block_number = int(client.eth.block_number) + return [load_adapter_state(client, adapter, block_number) for adapter in adapters] + + +# ---------------------------------------------------------------------------- +# Diff and alerting +# ---------------------------------------------------------------------------- + + +def _address_link(address: str, chain: Chain) -> str: + base = chain.explorer_url + return f"[{address}]({base}/address/{address})" if base else address + + +def _market_label(market_id: str, chain: Chain) -> tuple[str, int | None, str | None]: + """Return ``(markdown market link, loan decimals, loan symbol)``.""" + metadata = fetch_market_metadata(market_id, chain) + if metadata: + return ( + f"[{metadata['name']}]({get_market_url(market_id, chain)})", + metadata["loan_decimals"], + metadata["loan_symbol"], + ) + return f"[{market_id}]({get_market_url(market_id, chain)})", None, None + + +def _burn_body( + chain: Chain, + adapter: str, + burn: PendingBurn, + total_assets: int | None, + *, + reminder: bool, +) -> str: + market, decimals, symbol = _market_label(burn.market_id, chain) + title = "⏳ Reminder: adapter share burn is close" if reminder else "🔴 Adapter share burn submitted" + return ( + f"{title}: `burnShares` on market {market}\n" + f"🧩 Adapter: {_address_link(adapter, chain)}\n" + f"💸 At risk: {format_amount_at_risk(burn.assets, decimals, symbol, total_assets)}\n" + f"⏰ Executable at: {format_ts(burn.executable_at)} {format_countdown(burn.executable_at)}\n" + "Burning the shares writes off the adapter's supply in this market. " + "If unintended, the curator or a sentinel can call `revoke(data)` on the adapter." + ) + + +def _diff_burn_timelock(chain: Chain, state: AdapterTimelockState, diff: VaultDiff) -> None: + """Alert MEDIUM when ``timelock(burnShares)`` drops; the first run seeds the cache silently.""" + cache_key = _key(state.adapter, "burn_timelock", BURN_TIMELOCK_TYPE) + last = _read_str(cache_key) + if last and state.burn_timelock < int(last): + diff.alert( + AlertSeverity.MEDIUM, + f"🚨 Adapter {_address_link(state.adapter, chain)} `burnShares` timelock decreased: " + f"{format_timelock(int(last))} → {format_timelock(state.burn_timelock)}", + ) + if last != str(state.burn_timelock): + diff.defer(_write, cache_key, state.burn_timelock) + + +def _check_burn_loss(chain: Chain, total_assets: int | None, adapter: str, burn: PendingBurn, diff: VaultDiff) -> None: + """Alert once on a pending burn of held shares, then once more near ``executableAt``.""" + loss_key = _key(adapter, burn.market_id, PENDING_BURN_LOSS_TYPE) + reminder_key = _key(adapter, burn.market_id, PENDING_BURN_REMINDER_TYPE) + notice = due_loss_notice(burn.executable_at, _read_int(loss_key), _read_int(reminder_key)) + if notice is None: + return + diff.loss(_burn_body(chain, adapter, burn, total_assets, reminder=notice.reminder)) + diff.defer(_write, loss_key, burn.executable_at) + if notice.in_window: + diff.defer(_write, reminder_key, burn.executable_at) + + +def _diff_pending_burns(chain: Chain, total_assets: int | None, state: AdapterTimelockState, diff: VaultDiff) -> None: + """Alert on new pending burns (HIGH if the adapter holds shares), remind near the deadline, note resolutions. + + The loss alert is tracked on its own key, so a burn first seen with no shares + (LOW) escalates as soon as the adapter holds shares in that market. + """ + now = int(datetime.now().timestamp()) + current: set[str] = set() + for burn in state.pending_burns: + current.add(burn.market_id) + cache_key = _key(state.adapter, burn.market_id, PENDING_BURN_TYPE) + seen = _read_int(cache_key) == burn.executable_at + if burn.shares > 0: + _check_burn_loss(chain, total_assets, state.adapter, burn, diff) + elif not seen: + diff.alert( + AlertSeverity.LOW, + f"🔥 Adapter {_address_link(state.adapter, chain)} submitted `burnShares` on market " + f"[{burn.market_id}]({get_market_url(burn.market_id, chain)}) with no shares in it; " + f"executable at {format_ts(burn.executable_at)} {format_countdown(burn.executable_at)}", + ) + if not seen: + diff.defer(_write, cache_key, burn.executable_at) + + index_key = _key(state.adapter, "pending_burns", PENDING_BURN_INDEX_TYPE) + previous = {m for m in _read_str(index_key).split(",") if m} + for market_id in sorted(previous - current): + cache_key = _key(state.adapter, market_id, PENDING_BURN_TYPE) + executable_at = _read_int(cache_key) + verb = "executed" if 0 < executable_at <= now else "revoked" + diff.alert( + AlertSeverity.LOW, + f"{'✅' if verb == 'executed' else '🛑'} Pending `burnShares` on market " + f"[{market_id}]({get_market_url(market_id, chain)}) of adapter " + f"{_address_link(state.adapter, chain)} was {verb}.", + ) + # Clear the cursors so a later re-submit with the same executableAt still alerts. + for value_type in (PENDING_BURN_TYPE, PENDING_BURN_LOSS_TYPE, PENDING_BURN_REMINDER_TYPE): + diff.defer(_write, _key(state.adapter, market_id, value_type), 0) + diff.defer(_write, index_key, ",".join(sorted(current))) + + +def diff_adapter(chain: Chain, total_assets: int | None, state: AdapterTimelockState, diff: VaultDiff) -> None: + """Diff one adapter's on-chain timelock state against the cache into the vault's ``diff``.""" + _diff_burn_timelock(chain, state, diff) + _diff_pending_burns(chain, total_assets, state, diff) diff --git a/protocols/morpho/governance.py b/protocols/morpho/governance.py index d01c3b87..b40be198 100644 --- a/protocols/morpho/governance.py +++ b/protocols/morpho/governance.py @@ -4,14 +4,20 @@ from web3 import Web3 -from protocols.morpho._alerts import VaultDiff, send_vault_alerts +from protocols.morpho._alerts import VaultDiff, due_loss_notice, send_vault_alerts from protocols.morpho._shared import ( + MORPHO_BLUE_ABI_PATH, PROTOCOL, MorphoMonitoringError, execute_graphql, fetch_market_metadata, + format_amount_at_risk, + format_countdown, + format_timelock, + format_ts, get_market_url, get_vault_url, + to_assets_down, ) from protocols.morpho.config import VAULTS_V1_BY_CHAIN from utils.abi import load_abi @@ -29,7 +35,10 @@ PENDING_CAP_TYPE = "pending_cap" REMOVABLE_AT_TYPE = "removable_at" +REMOVAL_LOSS_TYPE = "removable_at_loss" +REMOVAL_REMINDER_TYPE = "removable_at_reminder" ABI_MORPHO = load_abi("protocols/morpho/abi/morpho.json") +ABI_MORPHO_BLUE = load_abi(MORPHO_BLUE_ABI_PATH) @dataclass(frozen=True) @@ -42,6 +51,11 @@ class MarketGovernanceState: pending_cap_timestamp: int current_cap: int removable_at: int + # The vault's own supply in the market on Morpho Blue. A removal that executes + # while this is non-zero writes the position off and the vault takes the loss. + supply_shares: int = 0 + supply_assets: int = 0 + vault_total_assets: int | None = None def fetch_pending_cap_market_ids(vault_address: str, chain: Chain) -> list[str]: @@ -151,20 +165,35 @@ def _load_market_governance_states( client: Any, block_number: int, ) -> list[MarketGovernanceState]: - """Batch pending-cap and config reads for V1 vault markets.""" + """Batch pending-cap, config, and vault-supply reads for V1 vault markets. + + None of these reads go through a market's IRM, so they can't revert on a broken + market. ``totalAssets`` does (it values every withdraw-queue market), so it is + read separately and only feeds the optional "% of vault" figure. + """ + morpho_address = morpho_contract.functions.MORPHO().call(block_identifier=block_number) + total_assets = _try_total_assets(morpho_contract, block_number) + blue = client.eth.contract(address=morpho_address, abi=ABI_MORPHO_BLUE) + + per_market = 4 with client.batch_requests() as batch: for market_id in market_ids: batch.add(morpho_contract.functions.pendingCap(market_id).call(block_identifier=block_number)) batch.add(morpho_contract.functions.config(market_id).call(block_identifier=block_number)) + batch.add(blue.functions.position(market_id, morpho_contract.address).call(block_identifier=block_number)) + batch.add(blue.functions.market(market_id).call(block_identifier=block_number)) responses = client.execute_batch(batch) - expected_count = len(market_ids) * 2 + expected_count = len(market_ids) * per_market if len(responses) != expected_count: - raise ValueError(f"Expected {expected_count} pendingCap/config responses, got {len(responses)}") + raise ValueError(f"Expected {expected_count} pendingCap/config/position/market responses, got {len(responses)}") states = [] for index, market_id in enumerate(market_ids): - pending_cap, pending_timestamp = responses[index * 2] - config = responses[index * 2 + 1] + offset = index * per_market + pending_cap, pending_timestamp = responses[offset] + config = responses[offset + 1] + supply_shares = responses[offset + 2][0] + market = responses[offset + 3] states.append( MarketGovernanceState( vault_address=morpho_contract.address, @@ -173,11 +202,23 @@ def _load_market_governance_states( pending_cap_timestamp=pending_timestamp, current_cap=config[0], removable_at=config[2], + supply_shares=supply_shares, + supply_assets=to_assets_down(supply_shares, market[0], market[1]), + vault_total_assets=total_assets, ) ) return states +def _try_total_assets(morpho_contract: Any, block_number: int) -> int | None: + """Best-effort ``totalAssets()``; None when it reverts (e.g. a withdraw-queue market's IRM reverts).""" + try: + return int(morpho_contract.functions.totalAssets().call(block_identifier=block_number)) + except Exception as e: + logger.warning("totalAssets() failed for vault %s, alerts omit the vault share: %s", morpho_contract.address, e) + return None + + def _vault_header(name: str, vault_address: str, chain: Chain) -> str: """One-line header for the vault's grouped alert.""" return f"[{name}]({get_vault_url(vault_address, chain)}) on {chain.name}" @@ -220,8 +261,37 @@ def _check_pending_cap(name: str, state: MarketGovernanceState, chain: Chain, di ) +def _removal_loss_body(state: MarketGovernanceState, chain: Chain, *, reminder: bool) -> str: + """Render the potential-loss section for a queued removal of a market the vault still supplies.""" + market_url = get_market_url(state.market_id, chain) + metadata = fetch_market_metadata(state.market_id, chain) + if metadata: + market_label = f"{metadata['name']} ({metadata['lltv'] / 1e18 * 100:.2f}%)" + decimals, symbol = metadata["loan_decimals"], metadata["loan_symbol"] + else: + market_label, decimals, symbol = state.market_id, None, None + at_risk = format_amount_at_risk(state.supply_assets, decimals, symbol, state.vault_total_assets) + title = "⏳ Reminder: market removal is close" if reminder else "🔴 Market removal queued with funds still supplied" + return ( + f"{title}\n" + f"Market: [{market_label}]({market_url})\n" + f"💸 At risk: {at_risk}\n" + f"⏰ Removable at: {format_ts(state.removable_at)} {format_countdown(state.removable_at)}\n" + "Removing the market writes off the remaining supply. Withdraw it before the removal, " + "or have the guardian/curator call `revokePendingMarketRemoval`." + ) + + def _check_market_removal(state: MarketGovernanceState, chain: Chain, diff: VaultDiff) -> None: - """Buffer a section for a newly queued V1 market removal.""" + """Buffer a section for a queued V1 market removal. + + A removal of a market the vault still supplies is a HIGH potential loss, for + any amount: once ``removableAt`` passes, ``updateWithdrawQueue`` can drop the + market and the supply leaves ``totalAssets``. It is re-alerted once inside the + reminder window, and a removal first seen on an empty market escalates as soon + as the vault supplies it. A removal of an empty market is routine cleanup and + stays MEDIUM. + """ if state.removable_at <= 0: return last_timestamp = get_last_executed_morpho_from_file( @@ -229,28 +299,47 @@ def _check_market_removal(state: MarketGovernanceState, chain: Chain, diff: Vaul state.market_id, REMOVABLE_AT_TYPE, ) - if state.removable_at <= last_timestamp: + seen = state.removable_at <= last_timestamp + if state.supply_shares > 0: + _check_removal_loss(state, chain, diff) + elif not seen: + market_url = get_market_url(state.market_id, chain) + market_name, _ = fetch_market_info(state.market_id, chain) + diff.alert( + AlertSeverity.MEDIUM, + f"➖ Queued to remove market [{market_name}]({market_url}) at {format_ts(state.removable_at)} " + "(no vault supply in the market)", + ) + else: logger.info( "Skipping previously alerted market removal for vault %s market %s", state.vault_address, state.market_id, ) - return + if not seen: + diff.defer( + write_last_executed_morpho_to_file, + state.vault_address, + state.market_id, + REMOVABLE_AT_TYPE, + state.removable_at, + ) - market_url = get_market_url(state.market_id, chain) - market_name, _ = fetch_market_info(state.market_id, chain) - removable_at = datetime.fromtimestamp(state.removable_at).strftime("%Y-%m-%d %H:%M:%S") - diff.alert( - AlertSeverity.MEDIUM, - f"➖ Queued to remove market [{market_name}]({market_url}) at {removable_at}", - ) - diff.defer( - write_last_executed_morpho_to_file, - state.vault_address, - state.market_id, - REMOVABLE_AT_TYPE, + +def _check_removal_loss(state: MarketGovernanceState, chain: Chain, diff: VaultDiff) -> None: + """Alert once on a queued removal of a supplied market, then once more near ``removableAt``.""" + vault, market = state.vault_address, state.market_id + notice = due_loss_notice( state.removable_at, + get_last_executed_morpho_from_file(vault, market, REMOVAL_LOSS_TYPE), + get_last_executed_morpho_from_file(vault, market, REMOVAL_REMINDER_TYPE), ) + if notice is None: + return + diff.loss(_removal_loss_body(state, chain, reminder=notice.reminder)) + diff.defer(write_last_executed_morpho_to_file, vault, market, REMOVAL_LOSS_TYPE, state.removable_at) + if notice.in_window: + diff.defer(write_last_executed_morpho_to_file, vault, market, REMOVAL_REMINDER_TYPE, state.removable_at) def check_market_governance_state(name: str, state: MarketGovernanceState, chain: Chain, diff: VaultDiff) -> None: @@ -279,13 +368,24 @@ def check_pending_role_change( role_type: str, timestamp: int, diff: VaultDiff, + message: str | None = None, ) -> None: + """Alert HIGH once per pending ``role_type`` change, keyed by its ``validAt`` timestamp.""" market_id = "" # use empty string for all markets because the value is used per vault if timestamp > get_last_executed_morpho_from_file(morpho_contract.address, market_id, role_type): - diff.alert(AlertSeverity.HIGH, f"🚨 {role_type.capitalize()} is changing") + diff.alert(AlertSeverity.HIGH, message or f"🚨 {role_type.capitalize()} is changing") diff.defer(write_last_executed_morpho_to_file, morpho_contract.address, market_id, role_type, timestamp) +def format_timelock_decrease(current: int, new: int, valid_at: int) -> str: + """Render a pending V1 timelock change; only decreases are ever pending on V1.""" + return ( + f"🚨 Timelock decrease: {format_timelock(current)} → {format_timelock(new)}\n" + f"⏰ Executable at: {format_ts(valid_at)} {format_countdown(valid_at)}\n" + "A shorter timelock shrinks the window to react to cap changes and market removals." + ) + + def check_timelock_and_guardian( morpho_contract: Any, client: Any, @@ -295,15 +395,20 @@ def check_timelock_and_guardian( with morpho_contract.w3.batch_requests() as batch: batch.add(morpho_contract.functions.pendingTimelock().call(block_identifier=block_number)) batch.add(morpho_contract.functions.pendingGuardian().call(block_identifier=block_number)) + batch.add(morpho_contract.functions.timelock().call(block_identifier=block_number)) responses = client.execute_batch(batch) - if len(responses) != 2: - raise ValueError(f"Expected 2 responses from batch, got {len(responses)}") - - timelock = responses[0][1] # [1] to get the timestamp - guardian = responses[1][1] # [1] to get the timestamp - - check_pending_role_change(morpho_contract, "timelock", timelock, diff) - check_pending_role_change(morpho_contract, "guardian", guardian, diff) + if len(responses) != 3: + raise ValueError(f"Expected 3 responses from batch, got {len(responses)}") + + pending_timelock_value, timelock_valid_at = responses[0] + guardian_valid_at = responses[1][1] # [1] to get the timestamp + current_timelock = responses[2] + + timelock_message = None + if timelock_valid_at > 0: + timelock_message = format_timelock_decrease(current_timelock, pending_timelock_value, timelock_valid_at) + check_pending_role_change(morpho_contract, "timelock", timelock_valid_at, diff, timelock_message) + check_pending_role_change(morpho_contract, "guardian", guardian_valid_at, diff) def get_data_for_chain(chain: Chain) -> None: diff --git a/protocols/morpho/governance_v2.py b/protocols/morpho/governance_v2.py index 3b0ef6f2..8f6582e6 100644 --- a/protocols/morpho/governance_v2.py +++ b/protocols/morpho/governance_v2.py @@ -13,10 +13,14 @@ * **Owner / curator changes** — instant, no timelock. * **Sentinels / allocators / adapters** — added or removed. -NOT covered: ``MorphoMarketV1AdapterV2``'s own internal timelock system. The -GraphQL API does not surface adapter-internal pending operations; replaying -their Submit/Accept/Revoke events would reintroduce the RPC cost we're -explicitly avoiding. Phase-2 candidate. +Pending operations that can write off vault funds are HIGH and sent as their +own message (see ``_alerts.LOSS_PREFIX``), then re-alerted once near their +execution time: ``removeAdapter`` of an adapter that still holds assets, and +``decreaseTimelock(removeAdapter)`` while any adapter holds assets. An operation +first seen with nothing at risk escalates once funds arrive. + +``MorphoMarketV1AdapterV2``'s own timelock (``burnShares``) is not exposed by the +GraphQL API; ``adapter_timelocks_v2`` reads it on-chain with a few batched calls. """ from __future__ import annotations @@ -27,16 +31,21 @@ from web3 import Web3 -from protocols.morpho._alerts import VaultDiff, send_vault_alerts +from protocols.morpho import adapter_timelocks_v2 +from protocols.morpho._alerts import VaultDiff, due_loss_notice, send_vault_alerts from protocols.morpho._shared import ( PROTOCOL, MorphoV2MonitoringError, execute_graphql, + format_amount_at_risk, + format_countdown, + format_ts, get_vault_url, require_configured_keys, ) +from protocols.morpho.adapter_timelocks_v2 import AdapterTimelockState from protocols.morpho.config import VAULTS_V2_BY_CHAIN, get_vault_query_config -from protocols.morpho.v2_decoders import decode_submit, submit_data_key +from protocols.morpho.v2_decoders import decode_call, decode_submit, selector_of, submit_data_key from utils.alert import AlertSeverity from utils.cache import ( get_last_value_for_key_from_file, @@ -55,6 +64,10 @@ PENDING_FUNCTION_TYPE = "v2_pending_function" ROLE_TYPE = "v2_role" SET_TYPE = "v2_set" +PENDING_LOSS_TYPE = "v2_pending_loss" +PENDING_REMINDER_TYPE = "v2_pending_reminder" + +REMOVE_ADAPTER_SELECTOR = selector_of("removeAdapter(address)") # Sentinel cache values for pending operations: # * positive int: validAt of the latest pending submission we've alerted on @@ -63,19 +76,24 @@ EXECUTED = -1 REVOKED = 0 +# Morpho caps query complexity at 1,000,000 and the page size multiplies the cost +# of every nested field: first: 200 measured 1,151,000, first: 100 measures 575,500. +GOVERNANCE_PAGE_SIZE = 100 + _GOVERNANCE_QUERY = """ -query GovernanceV2($addresses: [String!]!, $chainIds: [Int!]!) { - vaultV2s(first: 200, where: { address_in: $addresses, chainId_in: $chainIds }) { +query GovernanceV2($addresses: [String!]!, $chainIds: [Int!]!, $first: Int!) { + vaultV2s(first: $first, where: { address_in: $addresses, chainId_in: $chainIds }) { items { address name chain { id } + totalAssets asset { symbol decimals } owner { address } curator { address } sentinels { sentinel { address } } allocators { allocator { address } } - adapters { items { address type } } + adapters { items { address type assets } } pendingConfigs { items { validAt functionName data txHash } } @@ -121,6 +139,15 @@ class V2GovernanceSnapshot: # The vault's own asset — denominates absolute caps in decoded operations. asset_symbol: str = "" asset_decimals: int | None = None + total_assets: int | None = None + # Checksummed adapter address → assets it holds / GraphQL adapter type. + adapter_assets: Dict[str, int] = field(default_factory=dict) + adapter_types: Dict[str, str] = field(default_factory=dict) + + @property + def market_v1_adapters(self) -> List[str]: + """Adapters with their own timelock (``MorphoMarketV1AdapterV2``).""" + return sorted(a for a, t in self.adapter_types.items() if t == "MorphoMarketV1") # ---------------------------------------------------------------------------- @@ -140,6 +167,10 @@ def _asset_decimals(item: Dict[str, Any]) -> int | None: return int(raw) if raw is not None else None +def _optional_int(value: Any) -> int | None: + return int(value) if value is not None else None + + def _checksum_or_empty(value: str) -> str: if not value: return "" @@ -159,7 +190,7 @@ def fetch_governance_snapshots() -> Dict[Chain, List[V2GovernanceSnapshot]]: data = execute_graphql( _GOVERNANCE_QUERY, - {"addresses": addresses, "chainIds": chain_ids}, + {"addresses": addresses, "chainIds": chain_ids, "first": GOVERNANCE_PAGE_SIZE}, "Vault V2 governance", error_type=MorphoV2MonitoringError, ) @@ -179,7 +210,8 @@ def fetch_governance_snapshots() -> Dict[Chain, List[V2GovernanceSnapshot]]: sentinels = [_checksum_or_empty(s["sentinel"]["address"]) for s in (item.get("sentinels") or [])] allocators = [_checksum_or_empty(a["allocator"]["address"]) for a in (item.get("allocators") or [])] - adapters = [_checksum_or_empty(a["address"]) for a in ((item.get("adapters") or {}).get("items") or [])] + adapter_items = (item.get("adapters") or {}).get("items") or [] + adapters = [_checksum_or_empty(a["address"]) for a in adapter_items] pending = [ PendingConfig( valid_at=int(pc["validAt"]), @@ -203,6 +235,9 @@ def fetch_governance_snapshots() -> Dict[Chain, List[V2GovernanceSnapshot]]: pending_configs=pending, asset_symbol=(item.get("asset") or {}).get("symbol") or "", asset_decimals=_asset_decimals(item), + total_assets=_optional_int(item.get("totalAssets")), + adapter_assets={_checksum_or_empty(a["address"]): int(a.get("assets") or 0) for a in adapter_items}, + adapter_types={_checksum_or_empty(a["address"]): a.get("type") or "" for a in adapter_items}, ) ) @@ -238,27 +273,6 @@ def _write(key: str, value: Any) -> None: # ---------------------------------------------------------------------------- -def _format_ts(ts: int) -> str: - return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") - - -def _format_countdown(ts: int) -> str: - """Human-friendly time remaining until ``ts``, e.g. ``(3 days)``.""" - seconds = ts - int(datetime.now().timestamp()) - if seconds <= 0: - return "(now)" - days = seconds / 86400 - if days >= 1: - n = round(days) - return f"({n} day{'s' if n != 1 else ''})" - hours = seconds / 3600 - if hours >= 1: - n = round(hours) - return f"({n} hour{'s' if n != 1 else ''})" - n = max(round(seconds / 60), 1) - return f"({n} minute{'s' if n != 1 else ''})" - - def _explorer_link(chain: Chain, tx_hash: str) -> str: base = chain.explorer_url if not base or not tx_hash: @@ -295,6 +309,76 @@ def _vault_header(snapshot: V2GovernanceSnapshot) -> str: return f"V2 [{snapshot.name}]({get_vault_url(snapshot.address, snapshot.chain)}) on {snapshot.chain.name}" +def _held_assets(snapshot: V2GovernanceSnapshot) -> int: + return sum(snapshot.adapter_assets.values()) + + +def pending_loss(snapshot: V2GovernanceSnapshot, pc: PendingConfig) -> tuple[int, str] | None: + """Return ``(assets at risk, reason)`` for a pending operation that can write off funds, else None. + + * ``removeAdapter(a)`` while ``a`` holds assets: ``totalAssets`` is the sum of + adapter ``realAssets``, so removing it writes those assets off. + * ``decreaseTimelock(removeAdapter, …)`` while any adapter holds assets: it + shrinks the window to react to such a removal. + """ + decoded = decode_call(pc.data) + if decoded is None: + return None + name, args = decoded + if name == "removeAdapter": + assets = snapshot.adapter_assets.get(str(Web3.to_checksum_address(args[0])), 0) + if assets > 0: + return assets, "Removing an adapter that still holds assets writes them off; deallocate first." + if name == "decreaseTimelock" and bytes(args[0]) == REMOVE_ADAPTER_SELECTOR: + held = _held_assets(snapshot) + if held > 0: + return held, "Shortens the timelock on `removeAdapter` while adapters hold assets." + return None + + +def _loss_body( + snapshot: V2GovernanceSnapshot, + pc: PendingConfig, + operation_label: str, + loss: tuple[int, str], + *, + reminder: bool, +) -> str: + """Render the potential-loss section for a pending operation.""" + at_risk, reason = loss + title = "⏳ Reminder: loss-causing operation is close" if reminder else "🔴 Submitted loss-causing operation" + amount = format_amount_at_risk( + at_risk, snapshot.asset_decimals, snapshot.asset_symbol or None, snapshot.total_assets + ) + return ( + f"{title}: {operation_label}\n" + f"💸 At risk: {amount}\n" + f"⏰ Executable at: {format_ts(pc.valid_at)} {format_countdown(pc.valid_at)}\n" + f"🔗 Tx: {_explorer_link(snapshot.chain, pc.tx_hash)}\n" + f"{reason} If unintended, the curator or a sentinel can call `revoke(data)`." + ) + + +def _check_pending_loss( + snapshot: V2GovernanceSnapshot, + pc: PendingConfig, + operation_label: str, + loss: tuple[int, str], + diff: VaultDiff, +) -> None: + """Alert once on a pending operation that puts funds at risk, then once more near ``validAt``.""" + addr = snapshot.address.lower() + loss_key = morpho_key(addr, pc.data_hash, PENDING_LOSS_TYPE) + reminder_key = morpho_key(addr, pc.data_hash, PENDING_REMINDER_TYPE) + notice = due_loss_notice(pc.valid_at, _read_int(loss_key), _read_int(reminder_key)) + if notice is None: + return + diff.loss(_loss_body(snapshot, pc, operation_label, loss, reminder=notice.reminder)) + diff.defer(_write, loss_key, pc.valid_at) + if notice.in_window: + diff.defer(_write, reminder_key, pc.valid_at) + + def _alert_pending_new( snapshot: V2GovernanceSnapshot, pending: List[tuple[PendingConfig, str]], @@ -315,7 +399,7 @@ def _alert_pending_new( diff.alert( AlertSeverity.MEDIUM, f"📥 Submitted: {operation_label}\n" - f"⏰ Executable at: {_format_ts(pc.valid_at)} {_format_countdown(pc.valid_at)}\n" + f"⏰ Executable at: {format_ts(pc.valid_at)} {format_countdown(pc.valid_at)}\n" f"🔗 Tx: {_explorer_link(snapshot.chain, pc.tx_hash)}", ) return @@ -327,12 +411,12 @@ def _alert_pending_new( for _, operation_label in pending: lines.append(f" • {operation_label}") pc0 = pending[0][0] - lines.append(f"⏰ Executable at: {_format_ts(pc0.valid_at)} {_format_countdown(pc0.valid_at)}") + lines.append(f"⏰ Executable at: {format_ts(pc0.valid_at)} {format_countdown(pc0.valid_at)}") lines.append(f"🔗 Tx: {_explorer_link(snapshot.chain, pc0.tx_hash)}") else: for pc, operation_label in pending: lines.append(f" • {operation_label}") - lines.append(f" ⏰ Executable at: {_format_ts(pc.valid_at)} {_format_countdown(pc.valid_at)}") + lines.append(f" ⏰ Executable at: {format_ts(pc.valid_at)} {format_countdown(pc.valid_at)}") lines.append(f" 🔗 Tx: {_explorer_link(snapshot.chain, pc.tx_hash)}") diff.alert(AlertSeverity.MEDIUM, "\n".join(lines)) @@ -356,7 +440,7 @@ def _alert_pending_resolved( operation = f"`{function_name}()`" if function_name else f"`{data_hash[:10]}…`" diff.alert( AlertSeverity.LOW, - f"{icon} Pending operation {operation} was {verb} (was due {_format_ts(last_valid_at)}).", + f"{icon} Pending operation {operation} was {verb} (was due {format_ts(last_valid_at)}).", ) @@ -391,13 +475,19 @@ def _diff_pending(snapshot: V2GovernanceSnapshot, diff: VaultDiff) -> None: diff.defer(_write, _pending_function_key(snapshot, pc.data_hash), _operation_function_name(pc, operation_label)) cache_key = morpho_key(addr, pc.data_hash, PENDING_TYPE) last = _read_int(cache_key) - # Already alerted at this validAt, or marked executed. - if last == pc.valid_at or last == EXECUTED: + if last == EXECUTED: continue - new_pending.append((pc, operation_label)) - diff.defer(_write, cache_key, pc.valid_at) - - # Group all newly-submitted operations for this vault into one section. + # A loss is tracked on its own key, so an operation first seen with nothing + # at risk still escalates once funds arrive. + loss = pending_loss(snapshot, pc) + if loss is not None: + _check_pending_loss(snapshot, pc, operation_label, loss, diff) + elif last != pc.valid_at: + new_pending.append((pc, operation_label)) + if last != pc.valid_at: + diff.defer(_write, cache_key, pc.valid_at) + + # Group all other newly-submitted operations for this vault into one section. _alert_pending_new(snapshot, new_pending, diff) # Detect resolved entries: anything in last-run's index that isn't in the @@ -416,6 +506,8 @@ def _diff_pending(snapshot: V2GovernanceSnapshot, diff: VaultDiff) -> None: continue _alert_pending_resolved(data_hash, last, _read_str(_pending_function_key(snapshot, data_hash)), diff) diff.defer(_write, cache_key, EXECUTED if last <= int(datetime.now().timestamp()) else REVOKED) + for value_type in (PENDING_LOSS_TYPE, PENDING_REMINDER_TYPE): + diff.defer(_write, morpho_key(addr, data_hash, value_type), 0) diff.defer(_write, index_key, ",".join(sorted(current_keys))) @@ -444,7 +536,10 @@ def _diff_set(snapshot: V2GovernanceSnapshot, set_name: str, current: List[str], diff.defer(_write, cache_key, ",".join(sorted(current_set))) -def diff_and_alert(snapshot: V2GovernanceSnapshot) -> None: +def diff_and_alert( + snapshot: V2GovernanceSnapshot, + adapter_states: List[AdapterTimelockState] | None = None, +) -> None: """Diff a vault's snapshot against persisted state and emit one grouped alert. Every diff category (pending, owner/curator, sentinels/allocators/adapters) @@ -457,6 +552,9 @@ def diff_and_alert(snapshot: V2GovernanceSnapshot) -> None: than treating them as delivered. A partial send (part 1 of 2 lands, part 2 fails) therefore repeats the whole group next run — duplicates beat a governance change nobody ever sees. + + ``adapter_states`` are the on-chain adapter timelock reads; ``None`` skips the + adapter diff (e.g. when the RPC read failed) so its cache is left untouched. """ diff = VaultDiff() _diff_pending(snapshot, diff) @@ -465,6 +563,8 @@ def diff_and_alert(snapshot: V2GovernanceSnapshot) -> None: _diff_set(snapshot, "sentinels", snapshot.sentinels, diff) _diff_set(snapshot, "allocators", snapshot.allocators, diff) _diff_set(snapshot, "adapters", snapshot.adapters, diff) + for state in adapter_states or []: + adapter_timelocks_v2.diff_adapter(snapshot.chain, snapshot.total_assets, state, diff) send_vault_alerts(_vault_header(snapshot), diff.alerts, PROTOCOL) diff.commit() @@ -487,8 +587,15 @@ def main() -> None: if not vaults: continue for vault in vaults: + adapter_states: List[AdapterTimelockState] | None = None + try: + adapter_states = adapter_timelocks_v2.load_adapter_states(vault.chain, vault.market_v1_adapters) + except Exception as e: + # Still run the GraphQL diff; the adapter cache is left untouched until a read succeeds. + logger.exception("Failed to read adapter timelocks for %s on %s", vault.address, chain.name) + failures.append(f"{vault.name} adapters on {chain.name}: {type(e).__name__}: {e}") try: - diff_and_alert(vault) + diff_and_alert(vault, adapter_states) except Exception as e: logger.exception("Failed to process governance for %s on %s", vault.address, chain.name) failures.append(f"{vault.name} on {chain.name}: {type(e).__name__}: {e}") diff --git a/protocols/morpho/v2_decoders.py b/protocols/morpho/v2_decoders.py index 7a616f1f..1bd28b83 100644 --- a/protocols/morpho/v2_decoders.py +++ b/protocols/morpho/v2_decoders.py @@ -272,10 +272,8 @@ def _format_args( # noqa: PLR0911,PLR0912 """Render a decoded argument tuple per signature.""" name = _function_name(sig) - if name == "removeAdapter": - return "" - if name == "addAdapter": - return f"{_format_address(args[0])}" + if name in ("addAdapter", "removeAdapter"): + return _address_link(args[0], chain) if name == "setIsAllocator": addr, flag = args return f"{_format_address(addr)} = {bool(flag)}" @@ -370,6 +368,29 @@ def decode_submit( return f"{_function_name(sig)}({formatted})" +def decode_call(data: bytes) -> tuple[str, tuple[Any, ...]] | None: + """ABI-decode a known timelocked call into ``(function name, args)``, or None. + + Unlike ``decode_submit`` this returns raw arguments, so callers can make + severity decisions (e.g. which adapter a ``removeAdapter`` targets). + """ + if len(data) < 4: + return None + sig = SELECTOR_TO_SIG.get(_selector_hex(data)) + if sig is None: + return None + try: + args = abi_decode(_arg_types(sig), data[4:]) + except Exception: + return None + return _function_name(sig), tuple(args) + + +def selector_of(sig: str) -> bytes: + """Return the 4-byte selector for a solidity signature.""" + return bytes(Web3.keccak(text=sig)[:4]) + + def submit_data_key(data: bytes) -> str: """Stable cache key derived from Submit ``data`` (keccak hex, no 0x prefix).""" return bytes(Web3.keccak(data)).hex().lower() diff --git a/tests/test_morpho_adapter_timelocks_v2.py b/tests/test_morpho_adapter_timelocks_v2.py new file mode 100644 index 00000000..01043179 --- /dev/null +++ b/tests/test_morpho_adapter_timelocks_v2.py @@ -0,0 +1,250 @@ +"""Behavior tests for on-chain MorphoMarketV1AdapterV2 timelock monitoring.""" + +import unittest +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Iterator +from unittest.mock import MagicMock, patch + +from web3 import Web3 + +from protocols.morpho import adapter_timelocks_v2 as adapters +from protocols.morpho._alerts import VaultDiff +from protocols.morpho.adapter_timelocks_v2 import AdapterTimelockState, PendingBurn +from utils.alert import AlertSeverity +from utils.chains import Chain + +ADAPTER = Web3.to_checksum_address("0x" + "22" * 20) +MORPHO = Web3.to_checksum_address("0x" + "33" * 20) +MARKET_A = bytes.fromhex("ab" * 32) +MARKET_B = bytes.fromhex("cd" * 32) +MARKET_A_HEX = Web3.to_hex(MARKET_A) +EXECUTABLE_AT = 2_000_000_000 +METADATA = {"name": "WETH/USDC", "loan_symbol": "USDC", "loan_decimals": 6, "lltv": 860000000000000000} + + +def _state(burns: list[PendingBurn] | None = None, burn_timelock: int = 3 * 86400) -> AdapterTimelockState: + return AdapterTimelockState(adapter=ADAPTER, burn_timelock=burn_timelock, pending_burns=burns or []) + + +def _burn(shares: int, assets: int = 0, executable_at: int = EXECUTABLE_AT) -> PendingBurn: + return PendingBurn(market_id=MARKET_A_HEX, executable_at=executable_at, shares=shares, assets=assets) + + +class _Cache: + """In-memory stand-in for the morpho cache file.""" + + def __init__(self) -> None: + self.values: dict[str, str] = {} + + def patches(self) -> Any: + return ( + patch( + "protocols.morpho.adapter_timelocks_v2.get_last_value_for_key_from_file", + side_effect=lambda _f, key: self.values.get(key, 0), + ), + patch( + "protocols.morpho.adapter_timelocks_v2.write_last_value_to_file", + side_effect=lambda _f, key, value: self.values.__setitem__(key, str(value)), + ), + ) + + +class TestAdapterDiff(unittest.TestCase): + def setUp(self) -> None: + self.cache = _Cache() + + def _run(self, state: AdapterTimelockState, now: int = 1_000_000_000) -> VaultDiff: + diff = VaultDiff() + read, write = self.cache.patches() + with ( + read, + write, + patch("protocols.morpho.adapter_timelocks_v2.fetch_market_metadata", return_value=METADATA), + patch("protocols.morpho.adapter_timelocks_v2.datetime") as fake_datetime, + patch("protocols.morpho._alerts.datetime") as alerts_datetime, + ): + fake_datetime.now.return_value = datetime.fromtimestamp(now) + alerts_datetime.now.return_value = datetime.fromtimestamp(now) + adapters.diff_adapter(Chain.MAINNET, 10_000_000 * 10**6, state, diff) + diff.commit() + return diff + + def test_pending_burn_with_shares_is_high_then_reminded_once(self) -> None: + state = _state([_burn(shares=10**12, assets=2_500_000 * 10**6)]) + + first = self._run(state, now=EXECUTABLE_AT - 3 * 86400) + quiet = self._run(state, now=EXECUTABLE_AT - 2 * 86400) + reminder = self._run(state, now=EXECUTABLE_AT - 3600) + after = self._run(state, now=EXECUTABLE_AT - 60) + + self.assertEqual(len(first.alerts), 1) + alert = first.alerts[0] + self.assertEqual(alert.severity, AlertSeverity.HIGH) + self.assertTrue(alert.standalone) + self.assertIn("`burnShares` on market [WETH/USDC]", alert.body) + self.assertIn("2.50M USDC (25.00% of vault)", alert.body) + self.assertIn(f"[{ADAPTER}](", alert.body) + self.assertEqual(quiet.alerts, []) + self.assertEqual(len(reminder.alerts), 1) + self.assertIn("Reminder", reminder.alerts[0].body) + self.assertEqual(after.alerts, []) + + def test_shares_worth_zero_assets_are_still_a_loss(self) -> None: + """The loss alert follows shares, so a stale/zero valuation can't downgrade a real burn.""" + diff = self._run(_state([_burn(shares=1, assets=0)])) + + self.assertEqual(diff.alerts[0].severity, AlertSeverity.HIGH) + + def test_pending_burn_without_shares_is_low(self) -> None: + diff = self._run(_state([_burn(shares=0)])) + + self.assertEqual([a.severity for a in diff.alerts], [AlertSeverity.LOW]) + self.assertFalse(diff.alerts[0].standalone) + + def test_pending_burn_escalates_when_shares_go_from_zero_to_positive(self) -> None: + """A burn first seen without shares must escalate as soon as the adapter holds shares, + not wait for the reminder window (PR #436 review).""" + first = self._run(_state([_burn(shares=0)]), now=EXECUTABLE_AT - 3 * 86400) + escalated = self._run(_state([_burn(shares=10**12, assets=2_500_000 * 10**6)]), now=EXECUTABLE_AT - 2 * 86400) + repeat = self._run(_state([_burn(shares=10**12, assets=2_500_000 * 10**6)]), now=EXECUTABLE_AT - 2 * 86400 + 60) + reminder = self._run(_state([_burn(shares=10**12, assets=2_500_000 * 10**6)]), now=EXECUTABLE_AT - 3600) + + self.assertEqual([a.severity for a in first.alerts], [AlertSeverity.LOW]) + self.assertEqual(len(escalated.alerts), 1) + self.assertEqual(escalated.alerts[0].severity, AlertSeverity.HIGH) + self.assertTrue(escalated.alerts[0].standalone) + self.assertNotIn("Reminder", escalated.alerts[0].body) + self.assertEqual(repeat.alerts, []) + self.assertEqual(len(reminder.alerts), 1) + self.assertIn("Reminder", reminder.alerts[0].body) + + def test_resolved_burn_reports_executed_and_clears_its_cursor(self) -> None: + self._run(_state([_burn(shares=5)]), now=EXECUTABLE_AT - 86400 * 3) + diff = self._run(_state(), now=EXECUTABLE_AT + 60) + + self.assertEqual(len(diff.alerts), 1) + self.assertEqual(diff.alerts[0].severity, AlertSeverity.LOW) + self.assertIn("was executed", diff.alerts[0].body) + # A re-submit with the same executableAt alerts again. + again = self._run(_state([_burn(shares=5)]), now=EXECUTABLE_AT - 86400 * 3) + self.assertEqual(again.alerts[0].severity, AlertSeverity.HIGH) + + def test_burn_timelock_seeds_silently_then_alerts_medium_on_decrease(self) -> None: + seed = self._run(_state(burn_timelock=3 * 86400)) + increase = self._run(_state(burn_timelock=7 * 86400)) + decrease = self._run(_state(burn_timelock=0)) + + self.assertEqual(seed.alerts, []) + self.assertEqual(increase.alerts, []) + self.assertEqual(len(decrease.alerts), 1) + self.assertEqual(decrease.alerts[0].severity, AlertSeverity.MEDIUM) + self.assertIn("7d → 0s", decrease.alerts[0].body) + + +class _FakeBatch: + def __init__(self) -> None: + self.calls: list[Any] = [] + + def add(self, call: Any) -> None: + self.calls.append(call) + + +class _FakeClient: + """Minimal batching client whose contract calls only resolve inside a batch context. + + Mirrors web3's batching: a ``.call()`` made before entering ``batch_requests()`` + would run eagerly, so the fake rejects it. Only the listed functions exist, so + a call to ``expectedSupplyAssets`` (IRM-dependent, may revert) or ``get_logs`` + fails the test. + """ + + def __init__(self, values: dict[tuple[str, Any], Any]) -> None: + self.values = values + self.in_batch = False + names = ("marketIdsLength", "timelock", "morpho", "marketIds", "executableAt", "supplyShares") + functions = MagicMock(spec=list(names)) + for name in names: + setattr(functions, name, self._bound(name)) + blue_functions = MagicMock(spec=["market"]) + blue_functions.market = self._bound("market") + adapter = MagicMock(functions=functions, address=ADAPTER) + blue = MagicMock(functions=blue_functions) + self.eth = MagicMock(spec=["contract"]) + self.eth.contract.side_effect = lambda address, abi: adapter if address == ADAPTER else blue + + def _bound(self, name: str) -> Any: + def bind(*args: Any) -> Any: + def call(**_kwargs: Any) -> Any: + if not self.in_batch: + raise AssertionError(f"{name}.call() ran outside a batch context") + return self.values[(name, args[0] if args else None)] + + return MagicMock(call=call) + + return bind + + @contextmanager + def batch_requests(self) -> Iterator[_FakeBatch]: + self.in_batch = True + try: + yield _FakeBatch() + finally: + self.in_batch = False + + def execute_batch(self, batch: _FakeBatch) -> list[Any]: + return batch.calls + + +def _values(held: list[bytes], executable_at: dict[bytes, int], adapter_shares: dict[bytes, int]) -> dict: + values: dict[tuple[str, Any], Any] = { + ("marketIdsLength", None): len(held), + ("timelock", adapters.BURN_SHARES_SELECTOR): 259200, + ("morpho", None): MORPHO, + } + for index, market in enumerate(held): + values[("marketIds", index)] = market + for market in (MARKET_A, MARKET_B): + values[("executableAt", adapters.burn_shares_calldata(market))] = executable_at.get(market, 0) + values[("supplyShares", market)] = adapter_shares.get(market, 0) + values[("market", market)] = (2 * 10**6, 2 * 10**12, 0, 0, 0, 0) + return values + + +class TestLoadAdapterState(unittest.TestCase): + def setUp(self) -> None: + self.cache = _Cache() + + def _load(self, client: _FakeClient) -> AdapterTimelockState: + read, write = self.cache.patches() + with read, write: + return adapters.load_adapter_state(client, ADAPTER, block_number=1) + + def test_reads_canonical_pending_burns_and_values_only_those(self) -> None: + client = _FakeClient( + _values([MARKET_A, MARKET_B], {MARKET_A: EXECUTABLE_AT}, {MARKET_A: 2 * 10**12, MARKET_B: 2 * 10**12}) + ) + + state = self._load(client) + + self.assertEqual(state.burn_timelock, 259200) + self.assertEqual(state.pending_burns, [PendingBurn(MARKET_A_HEX, EXECUTABLE_AT, 2 * 10**12, 2_000_000)]) + + def test_no_pending_burn_needs_no_valuation_calls(self) -> None: + values = _values([MARKET_A], {}, {}) + del values[("morpho", None)] + client = _FakeClient(values) + + state = self._load(client) + + self.assertEqual(state.pending_burns, []) + + def test_burn_shares_calldata_matches_abi_encoding(self) -> None: + self.assertEqual( + adapters.burn_shares_calldata(MARKET_A), + bytes(Web3.keccak(text="burnShares(bytes32)")[:4]) + MARKET_A, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_morpho_governance.py b/tests/test_morpho_governance.py index 2671128b..8ded03bd 100644 --- a/tests/test_morpho_governance.py +++ b/tests/test_morpho_governance.py @@ -2,11 +2,12 @@ import unittest from typing import Any -from unittest.mock import patch +from unittest.mock import MagicMock, patch from protocols.morpho import governance from protocols.morpho._alerts import VaultDiff from protocols.morpho.governance import MarketGovernanceState +from utils.alert import AlertSeverity from utils.chains import Chain VAULT = "0x" + "11" * 20 @@ -69,6 +70,193 @@ def test_previously_alerted_market_removal_is_not_repeated(self) -> None: write.assert_not_called() +METADATA = {"name": "WETH/USDC", "loan_symbol": "USDC", "loan_decimals": 6, "lltv": 860000000000000000} +REMOVAL_STATE = dict(pending_cap=0, pending_cap_timestamp=0, current_cap=0) + + +class _Cache: + """In-memory stand-in for the morpho cache file, keyed like the real helpers.""" + + def __init__(self) -> None: + self.values: dict[tuple[str, str, str], int] = {} + + def get(self, vault: str, market: str, value_type: str) -> int: + return self.values.get((vault, market, value_type), 0) + + def write(self, vault: str, market: str, value_type: str, value: int) -> None: + self.values[(vault, market, value_type)] = value + + +class TestMorphoV1MarketRemovalLoss(unittest.TestCase): + def _run(self, state: MarketGovernanceState, cache: _Cache, now: int) -> VaultDiff: + diff = VaultDiff() + with ( + patch("protocols.morpho.governance.get_last_executed_morpho_from_file", side_effect=cache.get), + patch("protocols.morpho.governance.write_last_executed_morpho_to_file", side_effect=cache.write), + patch("protocols.morpho.governance.fetch_market_metadata", return_value=METADATA), + patch("protocols.morpho.governance.fetch_market_info", return_value=("WETH/USDC (86.00%)", 6)), + patch("protocols.morpho.governance.datetime") as fake_datetime, + patch("protocols.morpho._shared.datetime") as shared_datetime, + patch("protocols.morpho._alerts.datetime") as alerts_datetime, + ): + from datetime import datetime as real_datetime + + for fake in (fake_datetime, shared_datetime, alerts_datetime): + fake.now.return_value = real_datetime.fromtimestamp(now) + fake.fromtimestamp.side_effect = real_datetime.fromtimestamp + governance.check_market_governance_state("Example", state, Chain.MAINNET, diff) + diff.commit() + return diff + + def test_removal_with_supply_is_a_standalone_high_loss_alert(self) -> None: + removable_at = 2_000_000_000 + state = _state( + "0x" + "ab" * 32, + **REMOVAL_STATE, + removable_at=removable_at, + supply_shares=10**12, + supply_assets=1_500_000 * 10**6, + vault_total_assets=10_000_000 * 10**6, + ) + diff = self._run(state, _Cache(), now=removable_at - 3 * 86400) + + self.assertEqual(len(diff.alerts), 1) + alert = diff.alerts[0] + self.assertEqual(alert.severity, AlertSeverity.HIGH) + self.assertTrue(alert.standalone) + self.assertIn("funds still supplied", alert.body) + self.assertIn("1.50M USDC (15.00% of vault)", alert.body) + self.assertIn("revokePendingMarketRemoval", alert.body) + self.assertIn("(3 days)", alert.body) + + def test_any_non_zero_supply_is_high_and_dust_shows_raw_units(self) -> None: + state = _state("0x" + "ab" * 32, **REMOVAL_STATE, removable_at=2_000_000_000, supply_shares=1, supply_assets=1) + diff = self._run(state, _Cache(), now=1_000_000_000) + + self.assertEqual(diff.alerts[0].severity, AlertSeverity.HIGH) + self.assertIn("1 raw units USDC", diff.alerts[0].body) + + def test_removal_of_empty_market_stays_medium(self) -> None: + state = _state("0x" + "ab" * 32, **REMOVAL_STATE, removable_at=2_000_000_000) + diff = self._run(state, _Cache(), now=1_000_000_000) + + self.assertEqual(len(diff.alerts), 1) + self.assertEqual(diff.alerts[0].severity, AlertSeverity.MEDIUM) + self.assertFalse(diff.alerts[0].standalone) + self.assertIn("no vault supply", diff.alerts[0].body) + + def test_still_supplied_removal_is_reminded_once_inside_the_window(self) -> None: + removable_at = 2_000_000_000 + state = _state( + "0x" + "ab" * 32, **REMOVAL_STATE, removable_at=removable_at, supply_shares=5, supply_assets=5 * 10**6 + ) + cache = _Cache() + + first = self._run(state, cache, now=removable_at - 3 * 86400) + quiet = self._run(state, cache, now=removable_at - 2 * 86400) + reminder = self._run(state, cache, now=removable_at - 3600) + after = self._run(state, cache, now=removable_at - 60) + + self.assertEqual(len(first.alerts), 1) + self.assertEqual(quiet.alerts, []) + self.assertEqual(len(reminder.alerts), 1) + self.assertEqual(reminder.alerts[0].severity, AlertSeverity.HIGH) + self.assertIn("Reminder", reminder.alerts[0].body) + self.assertEqual(after.alerts, []) + + def test_first_alert_inside_the_window_suppresses_the_reminder(self) -> None: + removable_at = 2_000_000_000 + state = _state( + "0x" + "ab" * 32, **REMOVAL_STATE, removable_at=removable_at, supply_shares=5, supply_assets=5 * 10**6 + ) + cache = _Cache() + + first = self._run(state, cache, now=removable_at - 3600) + second = self._run(state, cache, now=removable_at - 60) + + self.assertEqual(len(first.alerts), 1) + self.assertEqual(second.alerts, []) + + def test_removal_first_seen_empty_escalates_once_the_vault_supplies_it(self) -> None: + removable_at = 2_000_000_000 + empty = _state("0x" + "ab" * 32, **REMOVAL_STATE, removable_at=removable_at) + supplied = _state( + "0x" + "ab" * 32, **REMOVAL_STATE, removable_at=removable_at, supply_shares=5, supply_assets=5 * 10**6 + ) + cache = _Cache() + + first = self._run(empty, cache, now=removable_at - 3 * 86400) + escalated = self._run(supplied, cache, now=removable_at - 2 * 86400) + repeat = self._run(supplied, cache, now=removable_at - 2 * 86400 + 3600) + emptied = self._run(empty, cache, now=removable_at - 2 * 86400 + 7200) + + self.assertEqual([a.severity for a in first.alerts], [AlertSeverity.MEDIUM]) + self.assertEqual(len(escalated.alerts), 1) + self.assertEqual(escalated.alerts[0].severity, AlertSeverity.HIGH) + self.assertTrue(escalated.alerts[0].standalone) + self.assertNotIn("Reminder", escalated.alerts[0].body) + self.assertEqual(repeat.alerts, []) + self.assertEqual(emptied.alerts, []) + + +class _V1Batch: + def __init__(self) -> None: + self.calls: list[Any] = [] + + def add(self, call: Any) -> None: + self.calls.append(call) + + +class TestMorphoV1VaultReads(unittest.TestCase): + def test_reverting_total_assets_only_drops_the_vault_share(self) -> None: + """totalAssets() values every withdraw-queue market and can revert on a broken IRM; + the removal check must still see supply and stay a loss alert.""" + from contextlib import contextmanager + + market_id = bytes.fromhex("ab" * 32) + vault = MagicMock(address=VAULT) + vault.functions.MORPHO.return_value.call.return_value = "0x" + "33" * 20 + vault.functions.totalAssets.return_value.call.side_effect = ValueError("execution reverted") + vault.functions.pendingCap.return_value.call.return_value = (0, 0) + vault.functions.config.return_value.call.return_value = (0, True, 2_000_000_000) + blue = MagicMock() + blue.functions.position.return_value.call.return_value = (10**12, 0, 0) + blue.functions.market.return_value.call.return_value = (2 * 10**6, 2 * 10**12, 0, 0, 0, 0) + client = MagicMock() + client.eth.contract.return_value = blue + + @contextmanager + def batch_requests() -> Any: + yield _V1Batch() + + client.batch_requests.side_effect = batch_requests + client.execute_batch.side_effect = lambda batch: batch.calls + + [state] = governance._load_market_governance_states(vault, [market_id], client, block_number=1) + + self.assertIsNone(state.vault_total_assets) + self.assertEqual( + (state.supply_shares, state.supply_assets, state.removable_at), (10**12, 1_000_000, 2_000_000_000) + ) + self.assertIn("1.00 USDC", governance.format_amount_at_risk(state.supply_assets, 6, "USDC", None)) + self.assertNotIn("of vault", governance.format_amount_at_risk(state.supply_assets, 6, "USDC", None)) + + +class TestMorphoV1Timelock(unittest.TestCase): + def test_timelock_decrease_message_shows_old_and_new_values(self) -> None: + message = governance.format_timelock_decrease(3 * 86400, 0, 2_000_000_000) + + self.assertIn("Timelock decrease: 3d → 0s", message) + self.assertIn("Executable at:", message) + + def test_to_assets_down_matches_shares_math(self) -> None: + # Virtual offsets: 1e6 shares for 1 asset on an empty market. + self.assertEqual(governance.to_assets_down(10**6, 0, 0), 1) + self.assertEqual(governance.to_assets_down(2 * 10**12, 2 * 10**6, 2 * 10**12), 2_000_000) + # Rounds down: 3 shares of a 10-asset / 1e7-share market are worth less than one unit. + self.assertEqual(governance.to_assets_down(3, 10, 10**7), 0) + + class TestMorphoV1GovernanceGrouping(unittest.TestCase): def test_findings_for_one_vault_collapse_into_a_single_message(self) -> None: """Two new markets on one vault produce one message, not two. diff --git a/tests/test_morpho_v2_decoders.py b/tests/test_morpho_v2_decoders.py index 2805604c..5f50749a 100644 --- a/tests/test_morpho_v2_decoders.py +++ b/tests/test_morpho_v2_decoders.py @@ -68,7 +68,7 @@ def test_remove_adapter(self): data = _build("removeAdapter(address)", ["address"], [A2]) self.assertEqual( decode_submit(data), - "removeAdapter()", + f"removeAdapter({Web3.to_checksum_address(A2)})", ) def test_set_is_allocator_grant(self): diff --git a/tests/test_morpho_v2_governance.py b/tests/test_morpho_v2_governance.py index 3d40f64b..3a86bba0 100644 --- a/tests/test_morpho_v2_governance.py +++ b/tests/test_morpho_v2_governance.py @@ -6,9 +6,11 @@ from web3 import Web3 from protocols.morpho import governance_v2 +from protocols.morpho._alerts import LOSS_PREFIX from protocols.morpho._shared import MorphoV2MonitoringError from protocols.morpho.governance_v2 import PendingConfig, V2GovernanceSnapshot from protocols.morpho.v2_decoders import submit_data_key +from utils.alert import AlertSeverity from utils.chains import Chain from utils.telegram import MAX_MESSAGE_LENGTH @@ -381,3 +383,161 @@ def test_fetch_fails_if_api_omits_configured_vaults(self) -> None: if __name__ == "__main__": unittest.main() + + +ADAPTER = Web3.to_checksum_address("0x" + "22" * 20) + + +def _loss_snapshot(pending_configs: list[PendingConfig], adapter_assets: int) -> V2GovernanceSnapshot: + snapshot = _snapshot(pending_configs) + snapshot.asset_symbol = "USDC" + snapshot.asset_decimals = 6 + snapshot.total_assets = 4_000_000 * 10**6 + snapshot.adapter_assets = {ADAPTER: adapter_assets} + snapshot.adapter_types = {ADAPTER: "MorphoMarketV1"} + return snapshot + + +def _pc(sig: str, types: list[str], values: list[Any], valid_at: int = 2_000_000_000) -> PendingConfig: + return PendingConfig( + valid_at=valid_at, + function_name=sig.split("(", 1)[0], + data=_build(sig, types, values), + tx_hash="0x" + "12" * 32, + ) + + +class TestMorphoV2LossClassification(unittest.TestCase): + def test_remove_adapter_holding_assets_is_a_loss_with_amount(self) -> None: + pc = _pc("removeAdapter(address)", ["address"], [ADAPTER]) + loss = governance_v2.pending_loss(_loss_snapshot([pc], 1), pc) + assert loss is not None + self.assertEqual(loss[0], 1) + + def test_remove_empty_adapter_is_not_a_loss(self) -> None: + pc = _pc("removeAdapter(address)", ["address"], [ADAPTER]) + self.assertIsNone(governance_v2.pending_loss(_loss_snapshot([pc], 0), pc)) + + def test_decrease_remove_adapter_timelock_with_held_assets_is_a_loss(self) -> None: + pc = _pc("decreaseTimelock(bytes4,uint256)", ["bytes4", "uint256"], [_selector("removeAdapter(address)"), 0]) + loss = governance_v2.pending_loss(_loss_snapshot([pc], 5), pc) + assert loss is not None + self.assertEqual(loss[0], 5) + self.assertIn("Shortens the timelock", loss[1]) + + def test_other_timelock_decrease_and_abdicate_are_not_losses(self) -> None: + decrease = _pc("decreaseTimelock(bytes4,uint256)", ["bytes4", "uint256"], [_selector("addAdapter(address)"), 0]) + abdicate = _pc("abdicate(bytes4)", ["bytes4"], [_selector("setIsAllocator(address,bool)")]) + snapshot = _loss_snapshot([], 5) + self.assertIsNone(governance_v2.pending_loss(snapshot, decrease)) + self.assertIsNone(governance_v2.pending_loss(snapshot, abdicate)) + + +class TestMorphoV2LossDelivery(unittest.TestCase): + def _run(self, snapshot: V2GovernanceSnapshot, state: dict[str, str], now: int) -> list[Any]: + from datetime import datetime as real_datetime + + sent: list[Any] = [] + with ( + patch( + "protocols.morpho.governance_v2.get_last_value_for_key_from_file", + side_effect=lambda _f, key: state.get(key, 0), + ), + patch( + "protocols.morpho.governance_v2.write_last_value_to_file", + side_effect=lambda _f, key, value: state.__setitem__(key, str(value)), + ), + patch("protocols.morpho._alerts.send_alert", side_effect=sent.append), + patch("protocols.morpho.governance_v2.datetime") as fake_datetime, + patch("protocols.morpho._alerts.datetime") as alerts_datetime, + ): + fake_datetime.now.return_value = real_datetime.fromtimestamp(now) + alerts_datetime.now.return_value = real_datetime.fromtimestamp(now) + governance_v2.diff_and_alert(snapshot) + return sent + + def test_loss_operation_is_sent_alone_and_reminded_once_near_deadline(self) -> None: + valid_at = 2_000_000_000 + remove = _pc("removeAdapter(address)", ["address"], [ADAPTER], valid_at) + add = _pc("addAdapter(address)", ["address"], [A1], valid_at) + snapshot = _loss_snapshot([remove, add], 1_000_000 * 10**6) + state: dict[str, str] = {} + + first = self._run(snapshot, state, now=valid_at - 3 * 86400) + quiet = self._run(snapshot, state, now=valid_at - 2 * 86400) + reminder = self._run(snapshot, state, now=valid_at - 3600) + after = self._run(snapshot, state, now=valid_at - 60) + + # The loss goes out on its own, ahead of the grouped routine message. + self.assertEqual(len(first), 2) + self.assertEqual(first[0].severity, AlertSeverity.HIGH) + self.assertTrue(first[0].message.startswith(LOSS_PREFIX)) + self.assertIn(f"removeAdapter([{ADAPTER}]", first[0].message) + self.assertIn("1.00M USDC (25.00% of vault)", first[0].message) + self.assertEqual(first[1].severity, AlertSeverity.MEDIUM) + self.assertNotIn("removeAdapter", first[1].message) + self.assertEqual(quiet, []) + self.assertEqual(len(reminder), 1) + self.assertIn("Reminder", reminder[0].message) + self.assertEqual(after, []) + + def test_remove_adapter_escalates_when_the_adapter_gets_funded(self) -> None: + """An operation first sent as MEDIUM (empty adapter) must escalate once the + adapter holds assets, not wait for the reminder window (PR #436 review).""" + valid_at = 2_000_000_000 + remove = _pc("removeAdapter(address)", ["address"], [ADAPTER], valid_at) + state: dict[str, str] = {} + + first = self._run(_loss_snapshot([remove], 0), state, now=valid_at - 3 * 86400) + escalated = self._run(_loss_snapshot([remove], 10**6), state, now=valid_at - 2 * 86400) + repeat = self._run(_loss_snapshot([remove], 10**6), state, now=valid_at - 2 * 86400 + 60) + + self.assertEqual([a.severity for a in first], [AlertSeverity.MEDIUM]) + self.assertEqual(len(escalated), 1) + self.assertEqual(escalated[0].severity, AlertSeverity.HIGH) + self.assertTrue(escalated[0].message.startswith(LOSS_PREFIX)) + self.assertNotIn("Reminder", escalated[0].message) + self.assertEqual(repeat, []) + + def test_abdicate_is_grouped_at_medium(self) -> None: + abdicate = _pc("abdicate(bytes4)", ["bytes4"], [_selector("setIsAllocator(address,bool)")]) + sent = self._run(_loss_snapshot([abdicate], 0), {}, now=1_000_000_000) + + self.assertEqual(len(sent), 1) + self.assertEqual(sent[0].severity, AlertSeverity.MEDIUM) + self.assertNotIn("POTENTIAL VAULT LOSS", sent[0].message) + + +class TestMorphoV2GovernanceMainAdapters(unittest.TestCase): + def test_adapter_read_failure_still_runs_graphql_diff_and_fails_the_run(self) -> None: + snapshot = _loss_snapshot([], 1) + diffed: list[Any] = [] + + with ( + patch( + "protocols.morpho.governance_v2.fetch_governance_snapshots", + return_value={Chain.MAINNET: [snapshot]}, + ), + patch( + "protocols.morpho.governance_v2.adapter_timelocks_v2.load_adapter_states", + side_effect=ValueError("No providers found for chain MAINNET"), + ) as load, + patch( + "protocols.morpho.governance_v2.diff_and_alert", + side_effect=lambda vault, states: diffed.append((vault, states)), + ), + self.assertRaisesRegex(MorphoV2MonitoringError, "adapters on MAINNET: ValueError"), + ): + governance_v2.main() + + load.assert_called_once_with(Chain.MAINNET, [ADAPTER]) + # The GraphQL diff still ran, with the adapter diff skipped. + self.assertEqual(diffed, [(snapshot, None)]) + + +class TestMorphoV2GovernanceQuery(unittest.TestCase): + def test_configured_vaults_fit_in_one_page(self) -> None: + """The query fetches a single page; outgrowing it would fail every run.""" + from protocols.morpho.config import VAULTS_V2_BY_CHAIN, iter_vaults + + self.assertLessEqual(len(list(iter_vaults(VAULTS_V2_BY_CHAIN))), governance_v2.GOVERNANCE_PAGE_SIZE) diff --git a/utils/chains.py b/utils/chains.py index c6469125..029248ef 100644 --- a/utils/chains.py +++ b/utils/chains.py @@ -15,6 +15,7 @@ # Setting any of those env vars disables the public fallback for that chain. PUBLIC_RPC_URLS: dict[int, str] = { 999: "https://rpc.hyperliquid.xyz/evm", + 4663: "https://rpc.mainnet.chain.robinhood.com", }