fix(security): redact secrets, harden dispatch, add timeouts, tighten deploy

.github/workflows/ci.yml

@@ -7,14 +7,20 @@ on:
     branches: [main]
   workflow_dispatch:
 
+# Read-only token: CI only checks out and tests.
+permissions:
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v10.2.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           enable-cache: true
 
@@ -30,10 +36,12 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v10.2.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           enable-cache: true
 
@@ -55,10 +63,12 @@ jobs:
   audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v10.2.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           enable-cache: true
 

automation/__main__.py

@@ -21,10 +21,13 @@
 def _configure_logging() -> None:
     import os
 
-    from utils.logger import quiet_dependency_loggers
+    from utils.logger import RedactingFormatter, quiet_dependency_loggers
 
     level = os.environ.get("LOG_LEVEL", "INFO").upper()
-    logging.basicConfig(level=level, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+    # Redacting: failed tasks' raw stderr (tracebacks quoting keyed URLs) is re-logged here.
+    handler = logging.StreamHandler()
+    handler.setFormatter(RedactingFormatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
+    logging.basicConfig(level=level, handlers=[handler])
     # Keep web3/urllib3 chatter out of the orchestrator logs; DEP_LOG_LEVEL re-enables it.
     quiet_dependency_loggers()
 

automation/config.py

@@ -15,6 +15,10 @@
 import yaml
 
 REPO_ROOT: Path = Path(os.environ.get("REPO_ROOT", Path(__file__).resolve().parent.parent))
+# Hard cap on one task's runtime. A hung task would otherwise hold the profile's
+# `flock` forever, so every later tick of that profile (and, for the sync profile,
+# code sync itself) is skipped. Override per task with `timeout:` (seconds).
+DEFAULT_TASK_TIMEOUT_SECONDS: int = 3600
 DEFAULT_JOBS_CONFIG_PATH: Path = REPO_ROOT / "automation" / "jobs.yaml"
 
 
@@ -28,6 +32,7 @@ class Task:
     script: str
     args: dict[str, str] = field(default_factory=dict)
     enabled: bool = True
+    timeout: int = DEFAULT_TASK_TIMEOUT_SECONDS
 
 
 @dataclass
@@ -55,7 +60,7 @@ def enabled_profiles(self) -> list[Profile]:
 
 
 _TASK_REQUIRED = ("name", "script")
-_TASK_ALLOWED = (*_TASK_REQUIRED, "args", "enabled")
+_TASK_ALLOWED = (*_TASK_REQUIRED, "args", "enabled", "timeout")
 _PROFILE_REQUIRED = ("cron",)
 _PROFILE_ALLOWED = (*_PROFILE_REQUIRED, "tasks", "env", "enabled", "description", "sync_before_run")
 
@@ -117,11 +122,16 @@ def _parse_task(profile: str, idx: int, body: Any, source: Path) -> Task:
         raise JobsConfigError(f"{source}: profile `{profile}` task #{idx}.args must be a mapping")
     args = {str(k): str(v) for k, v in args_raw.items()}
 
+    timeout = body.get("timeout", DEFAULT_TASK_TIMEOUT_SECONDS)
+    if isinstance(timeout, bool) or not isinstance(timeout, int) or timeout <= 0:
+        raise JobsConfigError(f"{source}: profile `{profile}` task #{idx}.timeout must be a positive integer")
+
     return Task(
         name=str(body["name"]),
         script=str(body["script"]),
         args=args,
         enabled=bool(body.get("enabled", True)),
+        timeout=timeout,
     )
 
 

automation/git_sync.py

@@ -33,6 +33,13 @@
 # Mirrors deploy/install.sh: the production venv carries the `ai` extra (LLM clients).
 UV_SYNC_ARGV: list[str] = ["uv", "sync", "--frozen", "--extra", "ai"]
 
+# Bounds on each step. A stalled `git fetch` (half-open TCP to GitHub) would
+# otherwise block the sync profile forever: its flock stays held, so every later
+# tick — and with it code sync — is skipped.
+GIT_FETCH_TIMEOUT_SECONDS = 120
+GIT_RESET_TIMEOUT_SECONDS = 60
+UV_SYNC_TIMEOUT_SECONDS = 600
+
 
 @dataclass
 class SyncResult:
@@ -61,6 +68,7 @@ def sync_to_remote_main(repo_root: Path) -> SyncResult:
             capture_output=True,
             text=True,
             check=False,
+            timeout=GIT_FETCH_TIMEOUT_SECONDS,
         )
         if fetch.returncode != 0:
             output = (fetch.stdout + fetch.stderr).strip()
@@ -71,7 +79,10 @@ def sync_to_remote_main(repo_root: Path) -> SyncResult:
             capture_output=True,
             text=True,
             check=False,
+            timeout=GIT_RESET_TIMEOUT_SECONDS,
         )
+    except subprocess.TimeoutExpired as exc:
+        return SyncResult(ok=False, output=f"git sync timed out after {exc.timeout}s: {' '.join(exc.cmd)}")
     except OSError as exc:
         return SyncResult(ok=False, output=f"git sync failed to spawn: {exc}")
 
@@ -116,7 +127,10 @@ def sync_dependencies(repo_root: Path) -> SyncResult:
             capture_output=True,
             text=True,
             check=False,
+            timeout=UV_SYNC_TIMEOUT_SECONDS,
         )
+    except subprocess.TimeoutExpired as exc:
+        return SyncResult(ok=False, output=f"uv sync timed out after {exc.timeout}s")
     except OSError as exc:
         return SyncResult(ok=False, output=f"uv sync failed to spawn: {exc}")
 

automation/jobs.yaml

@@ -15,6 +15,9 @@
 # `--key=value` in the declared order. Cache files resolve against $CACHE_DIR (set to
 # /srv/cache by the systemd unit; see utils/cache.py), so profiles only override a cache
 # *basename* when they need an isolated file (e.g. daily).
+#
+# Every task is killed after `timeout:` seconds (default 3600, see automation/config.py) so a
+# hung script can't hold its profile's flock forever; it then shows up in the failure digest.
 
 profiles:
 
@@ -111,11 +114,12 @@ profiles:
     # Tracked local edits are intentionally discarded in favor of reviewed main.
     # The sync also rewrites the live crontab when profiles or cadences change, and
     # a stale crontab calling a renamed profile still syncs (automation/__main__.py).
-    # NOTE: dependency changes (pyproject.toml/uv.lock) still need a manual
-    # `uv sync --frozen --extra ai` + `systemctl restart monitoring`.
+    # The sync also runs `uv sync --frozen --extra ai`, so dependency changes land
+    # with the code (a running monitoring-api still needs a restart to pick them up).
+    # stables gets a tight timeout: a hang here would stall the code-sync heartbeat.
     sync_before_run: true
     tasks:
-      - { name: "stables", script: protocols/stables/main.py }
+      - { name: "stables", script: protocols/stables/main.py, timeout: 300 }
 
   twenty_minute:
     # Every 20 min, offset 2 min so it never starts mid-sync of the ten_minute profile.

automation/runner.py

@@ -18,6 +18,7 @@
 
 from automation import crontab, git_sync
 from automation.config import Profile, Task
+from utils.redact import redact_secrets
 from utils.telegram import TelegramError, escape_markdown, send_telegram_message
 
 logger = logging.getLogger(__name__)
@@ -68,7 +69,8 @@ def _error_tail(stdout: str | None, stderr: str | None) -> str | None:
         lines = [ln.rstrip() for ln in stream.splitlines() if ln.strip()]
         if not lines:
             continue
-        tail = "\n".join(lines[-_ERROR_TAIL_LINES:])
+        # Tracebacks quote request URLs, which embed RPC/API keys.
+        tail = redact_secrets("\n".join(lines[-_ERROR_TAIL_LINES:]))
         if len(tail) > _ERROR_TAIL_CHARS:
             tail = "…" + tail[-_ERROR_TAIL_CHARS:]
         return tail
@@ -222,7 +224,26 @@ def _run_task(task: Task, *, profile: Profile, repo_root: Path, dry_run: bool) -
     logger.info("running task %s: %s", task.name, " ".join(argv))
     start = time.monotonic()
     try:
-        completed = subprocess.run(argv, cwd=repo_root, env=env, check=False, capture_output=True, text=True)
+        completed = subprocess.run(
+            argv, cwd=repo_root, env=env, check=False, capture_output=True, text=True, timeout=task.timeout
+        )
+    except subprocess.TimeoutExpired as exc:
+        duration = time.monotonic() - start
+        output = _decode_output(exc.stderr) + _decode_output(exc.stdout)
+        logger.error(
+            "task %s timed out after %ds and was killed%s",
+            task.name,
+            task.timeout,
+            f"\n{output.rstrip()}" if output.strip() else "",
+        )
+        tail = _error_tail(_decode_output(exc.stdout), _decode_output(exc.stderr))
+        return TaskResult(
+            name=task.name,
+            script=task.script,
+            returncode=-1,
+            duration_s=duration,
+            error=f"timed out after {task.timeout}s" + (f"\n{tail}" if tail else ""),
+        )
     except OSError as exc:
         duration = time.monotonic() - start
         logger.exception("task %s failed to spawn", task.name)
@@ -284,6 +305,15 @@ def _run_task(task: Task, *, profile: Profile, repo_root: Path, dry_run: bool) -
     )
 
 
+def _decode_output(output: str | bytes | None) -> str:
+    """Normalize partial output from ``TimeoutExpired`` (bytes even in text mode) to str."""
+    if output is None:
+        return ""
+    if isinstance(output, bytes):
+        return output.decode(errors="replace")
+    return output
+
+
 def _send_failure_digest(result: ProfileResult) -> None:
     message = result.telegram_summary()
     try:

deploy/Caddyfile

@@ -29,6 +29,13 @@
 # below, nothing to escape) and join multiple with `|`. Clients then call the
 # API with:  -H "Authorization: Bearer <their-token>"
 #
+# Tokens MUST be 64 hex characters: the config rejects any other bearer format
+# before comparing tokens, so an unset/empty ALERTS_API_TOKENS (which would turn
+# the token regex into `^Bearer ()$`) fails closed instead of matching an empty
+# token. Also refuse to start Caddy without tokens — in the same drop-in:
+#
+#   ExecStartPre=/bin/sh -c 'test -n "$ALERTS_API_TOKENS" || { echo "ALERTS_API_TOKENS unset" >&2; exit 1; }'
+#
 # Rate limiting (the commented `rate_limit` blocks) needs the caddy-ratelimit
 # plugin, which is NOT in the stock apt build. Either build it once with
 #   xcaddy build --with github.com/mholt/caddy-ratelimit
@@ -84,6 +91,13 @@
 
 	# --- token-gated API (everything else) ---
 	handle {
+		# Fail closed: only well-formed 64-hex bearer tokens are even considered,
+		# so an empty ALERTS_API_TOKENS can never authorize a request.
+		@malformed {
+			not header_regexp Authorization "^Bearer [0-9a-f]{64}$"
+		}
+		error @malformed "unauthorized" 401
+
 		# Accept any token in the pipe-separated ALERTS_API_TOKENS list.
 		@unauthorized {
 			not header_regexp Authorization "^Bearer ({$ALERTS_API_TOKENS})$"

deploy/alerts-api.md

@@ -221,8 +221,10 @@ you revoke one by dropping it from the list.
 openssl rand -hex 32     # generate one token per consumer (run once each)
 ```
 
-Use hex (`openssl rand -hex`) — it contains only `[0-9a-f]`, so it needs no
-escaping inside Caddy's matcher regex. ~32 bytes (256 bits) is plenty.
+Use exactly `openssl rand -hex 32` — 64 lowercase hex characters. The Caddyfile
+rejects any bearer value that isn't 64 hex characters before comparing tokens, so
+an empty or unset `ALERTS_API_TOKENS` fails closed; tokens in another format
+are refused. Hex also needs no escaping inside Caddy's matcher regex.
 
 **Where to store them:**
 

deploy/install.sh

@@ -24,13 +24,13 @@
 #      root:<deploy-user>) — copy from .env.example and fill in the values.
 # Then: `systemctl enable --now monitoring`.
 #
-# Private-repo auth over HTTPS resolves a token, in order:
-#   1. $GITHUB_TOKEN in the environment (pass with `sudo -E`).
-#   2. The invoking user's `gh` login.
-#   3. root's own `gh` login.
-# When found it's written to the target user's ~/.git-credentials (mode 600) so
-# the clone AND subsequent `git pull`s authenticate without prompting. SSH
-# remotes (git@github.com:...) skip this entirely.
+# Private-repo auth over HTTPS uses $GITHUB_TOKEN from the environment (pass
+# with `sudo -E`). It's written to the target user's ~/.git-credentials (mode
+# 600) so the clone AND the every-10-min sync authenticate without prompting.
+# That file is readable by the monitoring service itself, so use a fine-grained
+# token scoped to this one repository with read-only "Contents" access (or an SSH
+# deploy key, read-only) — never a personal `gh` login token, which usually
+# carries write and org scopes. SSH remotes (git@github.com:...) skip this entirely.
 
 set -Eeuo pipefail
 
@@ -77,24 +77,13 @@ as_user() {
   fi
 }
 
-# Resolve a GitHub token (env → invoking user's gh → root's gh). Prints the
-# token on stdout and returns 0 when found; returns 1 otherwise.
+# Print $GITHUB_TOKEN and return 0 when set; return 1 otherwise. Deliberately no
+# `gh auth token` fallback: that is the operator's personal, broadly scoped token,
+# and it would end up in a file the monitoring service can read.
 _github_token() {
   if [[ -n "${GITHUB_TOKEN:-}" ]]; then
     printf '%s' "$GITHUB_TOKEN"; return 0
   fi
-  if [[ -n "${SUDO_USER:-}" ]] && command -v gh >/dev/null 2>&1; then
-    local tok
-    if tok="$(sudo -u "$SUDO_USER" gh auth token 2>/dev/null)" && [[ -n "$tok" ]]; then
-      printf '%s' "$tok"; return 0
-    fi
-  fi
-  if command -v gh >/dev/null 2>&1; then
-    local tok
-    if tok="$(gh auth token 2>/dev/null)" && [[ -n "$tok" ]]; then
-      printf '%s' "$tok"; return 0
-    fi
-  fi
   return 1
 }
 
@@ -139,9 +128,10 @@ if [[ "$REPO_URL" == https://* ]]; then
     chmod 600 "$cred_file"
     as_user git config --global credential.helper store
   else
-    warn "no GITHUB_TOKEN / gh login found. If ${REPO_URL} is private the"
-    warn "clone below will fail. Re-run with a token, e.g.:"
-    warn "    sudo -E GITHUB_TOKEN=ghp_xxx bash deploy/install.sh"
+    warn "no GITHUB_TOKEN set. If ${REPO_URL} is private the clone below will"
+    warn "fail. Re-run with a fine-grained, read-only (Contents: read) token for"
+    warn "this repo only, e.g.:"
+    warn "    sudo -E GITHUB_TOKEN=github_pat_xxx bash deploy/install.sh"
   fi
 fi
 

deploy/runbook.md

@@ -146,6 +146,35 @@ The restart re-reads the env and re-renders the crontab. Anyone who could read
 the file has seen the old values, so a leaked credential should be rotated at
 the provider, not just edited here.
 
+`monitoring-api` does not load this file (it needs no credentials, and the unit
+makes `/etc/monitoring` inaccessible to it), so it needs no restart on rotation.
+
+Outbound text (logs, Telegram alerts, crash alerts, the automation digest and
+alert history) passes through `utils/redact.py`, which masks the values of
+`PROVIDER_URL_*` and every `*_KEY` / `*_TOKEN` / `*_SECRET` env var. Before that
+redaction existed, crash alerts quoted full RPC URLs, so **rotate any RPC key
+that may have appeared in an alert or log before the change**.
+
+---
+
+## Security baseline
+
+The box hard-resets to `origin/main` every 10 minutes and runs it with every
+secret in `/etc/monitoring/.env`, so write access to `main` is effectively code
+execution on the VPS. Keep these in place:
+
+- **Branch protection on `main`:** required PR review, required CI, no force
+  pushes or deletions, and ideally signed commits. Limit who can merge.
+- **Read-only repo credentials on the box:** a fine-grained token scoped to this
+  repo with `Contents: read` only, or a read-only SSH deploy key. `install.sh`
+  no longer copies a personal `gh` login token.
+- **Caddy:** tokens are 64-hex (`openssl rand -hex 32`); add the
+  `ExecStartPre` check from `deploy/Caddyfile` so Caddy refuses to start without
+  `ALERTS_API_TOKENS`.
+- **Emergency dispatch receiver:** the signed body now includes `sent_at` (unix
+  seconds). The liquidity-monitoring receiver should reject requests older than a
+  few minutes to stop replays.
+
 ---
 
 ## Test / staging run (route every alert to one dummy group)