diff --git a/apps/worker/package.json b/apps/worker/package.json index 86c5779..f817c99 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -7,9 +7,11 @@ "start": "bun src/main.js" }, "dependencies": { + "@profullstack/watchdog": "^0.2.0", "@tipoff/config": "workspace:*", "@tipoff/db": "workspace:*", "@tipoff/notify": "workspace:*", + "@tipoff/payments": "workspace:*", "@tipoff/queue": "workspace:*", "@tipoff/sports": "workspace:*" } diff --git a/apps/worker/src/main.js b/apps/worker/src/main.js index 4e51d5f..2e05dba 100644 --- a/apps/worker/src/main.js +++ b/apps/worker/src/main.js @@ -1,3 +1,4 @@ +import { watchDependencies } from '@profullstack/watchdog'; import { config } from '@tipoff/config'; /** * Workers on their own, for when one instance stops being enough. @@ -6,10 +7,10 @@ import { config } from '@tipoff/config'; * this entry exists so that splitting them is a Railway variable change and a * different start command, with no code to rewrite. */ -import { close as closeDb, sql } from '@tipoff/db'; +import { close as closeDb, healthcheck, sql } from '@tipoff/db'; import { migrate } from '@tipoff/db/migrate'; import { configurePayments } from '@tipoff/payments'; -import { closeQueues, installSchedules } from '@tipoff/queue'; +import { closeQueues, connection, installSchedules } from '@tipoff/queue'; import { startWorkers } from '@tipoff/queue/workers'; /* @@ -27,8 +28,27 @@ await migrate(); await installSchedules(); const workers = startWorkers(); +/* + * The same two watchdogs the combined entry runs. + * + * A worker has less to show for a wedge than a web process does: there is no + * page to hang, so a worker stuck on a client just stops doing the work and + * nothing anywhere goes red. That is worse, not better. This entry answers no + * requests at all, so there is not even a slow page to notice. + * + * The probes go through the shared `sql` handle and the shared `connection`, + * which is the whole trick: a second connection is the one thing guaranteed to + * look healthy while the real one is wedged. + */ +const watchdogs = watchDependencies({ + postgres: () => healthcheck(), + redis: () => connection.ping(), +}); + async function shutdown(signal) { console.log(`[worker] ${signal}, draining`); + // FIRST: a clean drain closes these clients and must not look like a wedge. + watchdogs.stop(); await Promise.allSettled(workers.map((w) => w.close())); await Promise.allSettled([closeQueues(), closeDb()]); process.exit(0);