From feaed974a79724e1fcbe067127b13e520829b54c Mon Sep 17 00:00:00 2001 From: kennyromanov Date: Wed, 7 Oct 2026 14:41:54 +0500 Subject: [PATCH 1/4] feat: The VoIP AI chatbot has been added --- .github/workflows/python-package.yml | 22 +- .gitignore | 2 +- AGENTS.md | 1482 +++++++++++++++++ README.md | 135 ++ docs/README.md | 134 ++ setup.py | 10 +- tests/calls/__init__.py | 0 tests/calls/conftest.py | 12 + tests/calls/test_call_recording.py | 77 + tests/calls/test_call_runtime.py | 23 + tests/calls/test_call_service.py | 427 +++++ tests/calls/test_call_state_machine.py | 178 ++ tests/calls/test_playback_state_machine.py | 43 + tests/calls/test_realtime_voice.py | 455 +++++ tests/test_calls_imports.py | 54 + whatsapp_chatbot_python/calls/__init__.py | 34 + whatsapp_chatbot_python/calls/contracts.py | 28 + whatsapp_chatbot_python/calls/models.py | 121 ++ .../calls/playback_state_machine.py | 46 + .../calls/realtime_voice.py | 520 ++++++ whatsapp_chatbot_python/calls/recording.py | 151 ++ whatsapp_chatbot_python/calls/runtime.py | 133 ++ whatsapp_chatbot_python/calls/service.py | 330 ++++ .../calls/state_machine.py | 194 +++ 24 files changed, 4606 insertions(+), 5 deletions(-) create mode 100644 AGENTS.md create mode 100644 tests/calls/__init__.py create mode 100644 tests/calls/conftest.py create mode 100644 tests/calls/test_call_recording.py create mode 100644 tests/calls/test_call_runtime.py create mode 100644 tests/calls/test_call_service.py create mode 100644 tests/calls/test_call_state_machine.py create mode 100644 tests/calls/test_playback_state_machine.py create mode 100644 tests/calls/test_realtime_voice.py create mode 100644 tests/test_calls_imports.py create mode 100644 whatsapp_chatbot_python/calls/__init__.py create mode 100644 whatsapp_chatbot_python/calls/contracts.py create mode 100644 whatsapp_chatbot_python/calls/models.py create mode 100644 whatsapp_chatbot_python/calls/playback_state_machine.py create mode 100644 whatsapp_chatbot_python/calls/realtime_voice.py create mode 100644 whatsapp_chatbot_python/calls/recording.py create mode 100644 whatsapp_chatbot_python/calls/runtime.py create mode 100644 whatsapp_chatbot_python/calls/service.py create mode 100644 whatsapp_chatbot_python/calls/state_machine.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 04d0336..9f90780 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,9 +30,25 @@ jobs: - name: Lint with ruff run: | # stop the build if there are Python syntax errors or undefined names - ruff check --output-format=github --select=E9,F63,F7,F82 --target-version=py37 . + ruff check --output-format=github --select=E9,F63,F7,F82 --target-version=py39 . # default set of ruff rules with GitHub Annotations - ruff check --output-format=github --target-version=py37 . + ruff check --output-format=github --target-version=py39 . - name: Test with pytest run: | - pytest + pytest tests --ignore=tests/calls + + calls: + name: VoIP / ${{ matrix.python-version }} + runs-on: ubuntu-24.04 + strategy: + matrix: + python-version: [ "3.11", "3.12", "3.13" ] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install optional call dependencies + run: python -m pip install '.[voip]' pytest + - name: Test calls and base imports + run: python -m pytest tests diff --git a/.gitignore b/.gitignore index 68bc17f..2dc53ca 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,4 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..9b53d24 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,1482 @@ +# sw-core + +Instructions for AI agents working with the repository. + +--- + +## 0) Quick facts about this repository + +* Purpose: a **runtime worker core** for managing live messenger instances through a custom WhatsApp server client. +* Current role: a lower-level runtime/service layer around `@green-api/sw-baileys`, responsible for live socket sessions, auth/session material, outbound commands, inbound events, webhook normalization, and local history. +* Current shape: + * Maintains local `WaInstance` runtimes + * Creates and supervises messenger sockets + * Stores auth/session state in Mongo + * Receives outbound commands through RabbitMQ and legacy REST paths + * Sends messages through live socket/runtime methods + * Receives messenger events from socket listeners + * Normalizes incoming/outgoing events into webhook/journal formats + * Publishes downstream webhook events through RabbitMQ + * Persists normalized message/chat/contact history in Mongo +* Stack: + * `TypeScript` + * `NestJS` + * `MongoDB` + * `RabbitMQ` + * Redis-like cache through `CacheFactory` + * `@green-api/sw-baileys` + * Custom Bayles/Baileys socket protocol +* Repository character: + * Legacy-heavy codebase + * Long-lived runtime sessions + * Stateful socket lifecycle + * Many explicit and implicit contracts + * Several externally observed behaviors that must not change casually + * Runtime behavior distributed across services, socket listeners, queues, caches, and persistence +* Key principle: this repository is **not** a clean standalone public API product, not a generic gateway, and not a place for broad architectural experiments. +* Architectural reality: many important contracts are implicit and historical. Queue names, message IDs, state changes, ACK/NACK behavior, retry behavior, webhook payloads, logs, caches, auth storage shape, and timing may be depended on by other services. +* Current core areas: + * `apps/swcore/src/app/wapi/wainstances/wa.instances.service.ts` — registry/supervisor of local Wa instances, lifecycle orchestration, outbound command bridge, reboot/delete flows + * `apps/swcore/src/app/wapi/wainstances/client/wa.instance.bayles.client.ts` — runtime of one Bayles instance: socket, auth state, reconnect, QR/pairing, event listeners, send methods, local caches + * `apps/swcore/src/app/wapi/controller/wa.controller.ts` — legacy REST surface for send, auth, QR, history, media, and instance operations + * `apps/swcore/src/app/wapi/wainstances/webhooks/webhook.service.bayles.ts` — incoming/outgoing event normalization, webhook queue publishing, journal/history mapping, dedupe/filter behavior + * `apps/swcore/src/app/wapi/wainstances/mapper/*` — webhook and journal mapper tree + * `apps/swcore/src/app/mongoauthstorage/authstore/use-multi-mongo-auth-state.ts` — Baileys auth state integration + * `apps/swcore/src/app/mongoauthstorage/authstore/auth.data/db/auth.data.mongo.service.ts` — auth data facade and storage routing + * `apps/swcore/src/app/wapi/rmq/*` — RabbitMQ topology, consumers, publishers, downloader integration + * `apps/swcore/src/app/wapi/jobs/jobs.ts` — periodic lifecycle/consumer jobs + * `apps/swcore/src/app/wapi/ws/qr.websocket.gateway.ts` — QR websocket gateway/cache behavior + +> Agents: before making changes, read the exact files involved in the affected flow. For lifecycle, outbound send, auth, webhook, or RabbitMQ changes, do not rely on one file only. Follow the runtime path end-to-end. + +--- + +## 1) Agent protocol + +1. **Understand the task first:** + * Determine whether the task changes: + * Instance lifecycle + * Socket/runtime behavior + * Authorization / QR / pairing flow + * Outbound send behavior + * RabbitMQ command handling + * REST legacy behavior + * Incoming event handling + * Webhook/journal normalization + * Mongo history persistence + * Auth/session storage + * Cache/reconnect behavior + * Media/downloader behavior + * Security/logging behavior + * Or only thin glue around existing behavior +2. **Assume legacy contracts exist:** + * If behavior is currently observed by external systems, treat it as a contract even when it is not documented. + * Do not change queue names, payload shapes, message ID semantics, status names, retry behavior, error behavior, delays, headers, or persistence shape without explicit intent. +3. **Trace the real runtime path:** + * For any change, identify where the flow starts and where it ends. + * Example: REST send starts in controller, goes through `WaInstancesService`, then into `WaInstanceBaylesClient`, then into Baileys socket, then later returns status through webhook events. + * Example: incoming message starts in socket listener, goes through `WebhookServiceBayles`, mapper tree, RabbitMQ publish, and Mongo history persistence. +4. **Minimal sufficient diff:** + * Modify only the files and sections required for the task. + * Minimal diff means the smallest complete safe change, including tests/docs/config when required. + * Do not perform broad cleanup, formatting sweeps, naming migrations, dependency upgrades, or framework reshaping unless explicitly required. +5. **Preserve runtime stability:** + * This service manages live sessions. A harmless-looking refactor can cause reconnect loops, lost events, duplicate webhooks, auth loss, delayed sends, or queue backlogs. +6. **Respect external dependencies:** + * SAPI/WaAdminJobs, RabbitMQ producers/consumers, Mongo collections, Media API, downloader service, Redis/cache, and downstream webhook consumers may depend on current behavior. +7. **Prefer explicit local fixes:** + * Add small named helpers when they clarify a concrete path. + * Avoid hiding runtime flow behind abstractions that make debugging harder. +8. **Avoid protocol invention:** + * Do not invent new lifecycle states, queue protocols, message ID formats, outbox semantics, delivery semantics, or auth storage categories unless explicitly requested. +9. **Document behavior changes:** + * If a real contract changes, update relevant docs, examples, configuration notes, and migration guidance. +10. **Be honest about uncertainty:** + * If a behavior appears legacy, implicit, or externally consumed, say so in the change summary. + * Do not present guesses as facts. + +--- + +## 2) Architecture and code placement + +### 2.1. Service shape + +This repository is best understood as: + +* A runtime-worker layer +* A local supervisor of live messenger instances +* A socket/session manager around Baileys +* A bridge between outbound command queues and live messenger sockets +* A normalizer between raw messenger events and downstream webhook/journal contracts +* A persistence layer for auth/session material and normalized history + +It is **not** best understood as: + +* A clean REST gateway +* A stateless controller-only service +* A generic public API product +* A full conversation platform +* A business workflow service +* A safe place for speculative architecture changes +* A repository where current behavior can be “cleaned up” without checking legacy consumers + +### 2.2. Current code placement + +**`apps/swcore/src/app/wapi/wainstances/wa.instances.service.ts`:** + +* Registry of local `WaInstanceBaylesClient` objects +* Instance lifecycle orchestration +* Startup initialization +* Reboot/spawn/delete flows +* Outbound command processing from RabbitMQ +* REST-to-runtime bridge methods +* Per-instance send mutex behavior +* Consumer start/stop coordination +* Some retry/failure handling around outbound messages + +**`apps/swcore/src/app/wapi/wainstances/client/wa.instance.bayles.client.ts`:** + +* Runtime for one messenger instance +* Baileys socket creation +* Connection lifecycle +* Reconnect behavior +* QR and pairing events +* Auth state usage +* Socket event listeners +* Send methods +* Local caches and reconnect slots +* State/status transitions +* Calls into webhook handling + +**`apps/swcore/src/app/wapi/controller/wa.controller.ts`:** + +* Legacy REST surface +* Direct runtime send path +* QR/pairing endpoints +* History endpoints +* Media/download endpoints +* Instance operation endpoints + +**`apps/swcore/src/app/wapi/wainstances/webhooks/webhook.service.bayles.ts`:** + +* Main incoming/outgoing event pipeline +* Webhook conversion +* Journal conversion +* Message status processing +* Dedupe-ish and filter behavior +* RabbitMQ webhook queue publishing +* History persistence coordination +* Event logging behavior + +**`apps/swcore/src/app/wapi/wainstances/mapper/interfaces.ts`:** + +* Public-ish enums and payload interfaces +* State/account/status/message/webhook types +* Many values here are externally observed and must be treated as contract-like + +**`apps/swcore/src/app/wapi/wainstances/mapper/webhook/*`:** + +* Webhook payload mapping +* Incoming/outgoing/call/avatar/state payload shapes +* Downstream compatibility concerns + +**`apps/swcore/src/app/wapi/wainstances/mapper/journal/*`:** + +* Journal/history mapping +* Mongo persistence compatibility concerns + +**`apps/swcore/src/app/mongoauthstorage/authstore/use-multi-mongo-auth-state.ts`:** + +* Integration point between Baileys auth state and Mongo auth storage +* `creds`, `keys.get`, `keys.set`, and `saveCreds` behavior +* Very sensitive protocol boundary + +**`apps/swcore/src/app/mongoauthstorage/authstore/auth.data/db/auth.data.mongo.service.ts`:** + +* Facade for auth data persistence +* Routes auth categories to sessions, prekeys, base categories, and creds services +* Storage shape compatibility matters + +**`apps/swcore/src/app/mongoauthstorage/model/*`:** + +* Mongo entities for auth/session material and normalized history +* Collection names and field shapes may be contract-like + +**`apps/swcore/src/app/wapi/rmq/topology/*`:** + +* Queue naming and topology +* Do not rename or reshape casually + +**`apps/swcore/src/app/wapi/rmq/rmq.consumer.service.ts`:** + +* Consumer lifecycle setup +* Instance-specific outgoing command consumption + +**`apps/swcore/src/app/wapi/rmq/rabbit/wa.message.consumer.ts`:** + +* RabbitMQ low-level consume behavior +* `prefetch`, ACK/NACK, and handler invocation + +**`apps/swcore/src/app/wapi/rmq/rmq.send.service.ts`:** + +* RabbitMQ publish path for downstream webhook events + +**`apps/swcore/src/app/wapi/rmq/rmq.downloader.service.ts`:** + +* Downloader queue integration for media events + +**`apps/swcore/src/app/wapi/ws/qr.websocket.gateway.ts`:** + +* QR gateway/cache/websocket delivery +* QR status compatibility concerns + +**`apps/swcore/src/app/wapi/jobs/jobs.ts`:** + +* Periodic jobs +* Consumer/reconnect/lifecycle behavior + +**`apps/swcore/src/app/common/factory/CacheFactory.ts`:** + +* Cache connection creation +* Sensitive logging risk +* Redis-like runtime dependency + +### 2.3. Main integration point + +The main integration point is **not** a controller method. It is the runtime relation between: + +```text +WaInstancesService + -> WaInstanceBaylesClient + -> @green-api/sw-baileys socket + -> socket events + -> WebhookServiceBayles + -> mapper tree + -> RabbitMQ / Mongo +``` + +Requirements: + +* Keep this flow traceable. +* Do not hide key lifecycle decisions behind generic infrastructure abstractions. +* Do not duplicate behavior that belongs to Baileys, the messenger protocol, or existing mapper logic. +* Preserve local runtime semantics unless explicitly changing them. + +### 2.4. Legacy reality + +This repository contains significant legacy behavior. + +Requirements: + +* Treat strange-looking code as potentially intentional until proven otherwise. +* Search for downstream usage before changing payloads, IDs, enums, statuses, queue names, or persistence shape. +* Avoid “cleanup” changes that alter timing, ordering, retries, logs, side effects, or fallback behavior. +* Do not replace legacy behavior with a cleaner model unless the task explicitly asks for a migration. +* If a change intentionally breaks legacy behavior, make that explicit in the summary and docs. + +--- + +## 3) Instance lifecycle and runtime state + +### 3.1. Core runtime objects + +The important runtime objects are: + +```typescript +export class WaInstanceBaylesClient extends EventEmitter { + public sock: WASocket | undefined = undefined + private statusInstance: StatusInstance = StatusInstance.Offline + private spawning = false + private attemptToConnect = 0 + private breakingDisconnectReason: BreakingDisconnectReason | undefined = undefined + private restartReason: number = undefined + private stateAccount: StateAccount +} +``` + +The service has two different state layers: + +```typescript +export enum StateAccount { + NotAuthorized = 'notAuthorized', + Authorized = 'authorized', + Blocked = 'blocked', + Starting = 'starting', + PendingCode = 'pendingCode', + YellowCard = 'yellowCard' +} + +export enum StatusInstance { + Online = 'online', + Offline = 'offline' +} +``` + +Requirements: + +* Do not merge `StateAccount` and `StatusInstance`. +* Do not assume `Authorized` means websocket is currently online. +* Do not assume `Online` means the account is semantically healthy. +* Preserve the distinction between auth/business state and live socket state. +* Treat state names as externally observed values. + +### 3.2. No clean FSM + +There are state enums, but there is no complete formal state machine. + +State transitions are distributed across: + +* `connection.update` +* `startAuthorization` +* `sendAuthorizationCode` +* `consumerJob` +* `logout` +* `resetSession` +* Reboot/spawn flows +* Reconnect handling +* Error branches + +Requirements: + +* Do not introduce a new FSM casually. +* Do not centralize state transitions as a cleanup unless explicitly requested and carefully tested. +* When changing state behavior, inspect all callers and downstream webhook behavior. +* State changes may emit webhooks, not just mutate local fields. + +### 3.3. Startup lifecycle + +At service startup: + +* `WaInstancesService.onModuleInit()` subscribes to SAPI updates and starts schedulers. +* Instance lists normally come from WaAdminJobs/SAPI. +* `initInstances()` creates local `WaInstanceBaylesClient` objects in chunks. +* Existing auth state in Mongo can trigger reboot/spawn behavior. +* Consumer startup may depend on authorization state. + +Requirements: + +* Do not assume all instances are present at process start. +* Do not assume local memory is the source of truth. +* Do not bypass SAPI/cache-api driven settings. +* Do not introduce pod-level ownership assumptions unless explicitly designed. +* Be careful with chunking, scheduling, and reboot behavior. + +### 3.4. Reconnect behavior + +Reconnect/supervision is not a single separate process. It is distributed across: + +* `connection.update` logic +* Periodic jobs +* Mutexes on reboot/connect/send +* Shared cache reconnect slots +* Local instance Map +* Status/account state transitions + +Requirements: + +* Do not remove or bypass reconnect guards because they appear redundant. +* Do not change reconnect delay/slot behavior without understanding multi-instance runtime effects. +* Do not add silent reconnect fallbacks that mask real auth/protocol failures. +* Do not assume there is a strong lease/heartbeat ownership model between pods unless you have verified it. + +--- + +## 4) Authorization, QR, pairing, and auth storage + +### 4.1. QR flow + +QR flow spans multiple layers: + +* REST QR endpoint +* QR websocket gateway/cache +* Service-level scan startup +* Baileys `connection.update` +* Socket event emission +* State/status changes + +Typical QR behavior: + +```typescript +if (qr) { + await this.changeStatusInstance(StatusInstance.Offline) + const qrImage = (await QRCode.toDataURL(qr)).replace('data:image/png;base64,', '') + this.emit('qr', { type: QRStatus.qrCode, message: qrImage }) + return +} +``` + +Requirements: + +* Do not change QR payload shape casually. +* Do not change QR cache/websocket behavior without checking REST and websocket consumers. +* Do not assume QR generation is independent of socket lifecycle. +* Preserve state/status side effects unless explicitly changing auth flow. + +### 4.2. Pairing code flow + +Pairing/mobile/phone-code flow uses separate endpoints and Baileys calls such as: + +```typescript +requestRegistrationCode(/* ... */) +register(/* ... */) +requestPairingCode(/* ... */) +``` + +Requirements: + +* Do not collapse QR and pairing code flow into one generic auth path without explicit design. +* Preserve existing mobile/phone-code semantics. +* Be careful with retry, state, and error behavior around pending code states. + +### 4.3. Successful authorization + +Successful authorization is detected through socket-level events such as: + +```typescript +connection === 'open' || isNewLogin +``` + +and then usually leads to: + +```typescript +await this.changeStatusInstance(StatusInstance.Online) +await this.changeStateAccount(StateAccount.Authorized) +await this.updateOnlineStatus() +``` + +Requirements: + +* Do not mark accounts authorized based only on REST request success. +* Do not bypass socket-confirmed auth transitions. +* Preserve webhook side effects of state changes. +* Be careful with `isNewLogin` behavior. + +### 4.4. Auth/session storage + +Auth/session/crypto state is stored through Mongo auth store integration. + +Important collections include: + +* `auth_creds` +* `auth_sessions` +* `auth_pre_keys` +* `auth_sender_keys` +* `auth_app_state_sync_keys` + +The Baileys auth state integration includes behavior like: + +```typescript +state: { + creds: authData.creds, + keys: { + get: async (category, ids) => ... + set: async (data) => ... + } +}, +saveCreds: async () => authDataService.saveAuthCreds(idInstance, authData.creds) +``` + +Requirements: + +* Treat auth storage as a protocol boundary. +* Do not change key categories, collection routing, serialization, or save timing casually. +* Do not rename auth collections or fields without migration planning. +* Do not add local encryption/decryption behavior without understanding infrastructure-level encryption and Baileys expectations. +* Do not log auth material, sessions, prekeys, sender keys, creds, or raw auth JSON. + +--- + +## 5) Outbound commands and send behavior + +### 5.1. There are two outbound paths + +There are two important send paths: + +1. **Legacy REST path** + * Controller calls runtime directly. + * It waits for runtime send promise. + * It usually returns `idMessage`. + * It is not an outbox API. + +2. **RabbitMQ command path** + * Commands arrive through `sw-outgoing-{id}`. + * Consumer parses `ProducerMessage`. + * Service dispatches by `producerMessage.type`. + * Runtime send uses `producerMessage.messageID`. + * ACK/NACK behavior controls retry/failure semantics. + +Requirements: + +* Do not merge REST and RabbitMQ semantics casually. +* Do not make REST send asynchronous/outbox-like unless explicitly requested. +* Do not remove RabbitMQ retry behavior when changing send methods. +* Do not assume send success means delivered/read. + +### 5.2. REST send semantics + +Typical REST path: + +```typescript +const idMessage = await this.waInstancesService.sendMessage(...) +res.status(HttpStatus.OK).json({ idMessage }) +``` + +Service path: + +```typescript +const waInstance = await this.throwWaInstance(idInstance) +return waInstance.sendMessage(...) +``` + +Runtime path eventually calls Baileys socket send methods. + +Requirements: + +* Preserve synchronous REST response semantics unless intentionally changing the public behavior. +* Preserve response shape such as `{ idMessage }` when existing clients depend on it. +* Do not add queue indirection to REST send as a cleanup. +* Do not report delivery/read status from REST send unless the protocol actually provides it at that time. + +### 5.3. RabbitMQ command semantics + +Typical command handling: + +```typescript +case TypesMessage.SendMessage: + await waInstance.sendMessage(..., producerMessage.messageID, ...) + break + +case TypesMessage.SendFileByUrl: + await waInstance.sendFileByUrl(..., producerMessage.messageID, ...) + break +``` + +Requirements: + +* Preserve queue names such as `sw-outgoing-{id}`. +* Preserve `ProducerMessage` shape unless explicitly changing producer/consumer contracts together. +* Preserve `producerMessage.messageID` semantics. +* Preserve ACK/NACK behavior. +* Preserve failure webhook behavior after retry exhaustion. +* Preserve per-instance send mutex unless explicitly redesigned. + +### 5.4. Message ID contracts + +Message IDs are not just arbitrary strings. + +Current behavior includes: + +* REST path may generate IDs internally. +* Some upload paths use `MessageIdService.generateID`. +* RabbitMQ path uses `producerMessage.messageID`. +* “Sent through API” classification may depend on structurally recognizable IDs. + +Requirements: + +* Do not change ID generation or parsing casually. +* Do not normalize IDs in ways that break downstream classification. +* Do not replace existing ID generation with UUIDs or random IDs unless explicitly required. +* Treat message ID structure as contract-like. + +### 5.5. Send success vs delivery status + +Send success means: + +* Runtime accepted/sent through socket +* A message ID was returned or reused +* It does **not** mean delivered +* It does **not** mean read +* Later status events produce webhook/status updates + +Requirements: + +* Do not rename send success into delivery success. +* Do not collapse send response and outgoing status webhook into one concept. +* Preserve separate outgoing status pipeline. + +### 5.6. Retry, throttling, and ordering + +RabbitMQ send path includes: + +* NACK for temporary errors +* ACK after final failure webhook +* Per-instance mutex +* Send delay through `delaySendMessagesMilliseconds` +* `MAX_DELAY_SEND_MESSAGES` +* Consumer `prefetch(1)` + +Requirements: + +* Treat retry and throttling as runtime behavior, not incidental implementation detail. +* Do not change `prefetch(1)` casually. +* Do not remove sleep/delay behavior as “unnecessary”. +* Do not convert per-instance serialized sends into parallel sends without explicit design. +* Be careful with timeout, socket-not-open, Mongo, uploader/downloader, and temporary infrastructure error branches. + +--- + +## 6) Incoming events, webhooks, journal, and history + +### 6.1. Socket listeners + +Runtime socket listeners are created after socket spawn and include events such as: + +```typescript +this.sock.ev.on('messages.upsert' /*, ...*/) +this.sock.ev.on('messages.update' /*, ...*/) +this.sock.ev.on('message-receipt.update' /*, ...*/) +this.sock.ev.on('presence.update' /*, ...*/) +this.sock.ws.on('CB:ack,class:message' /*, ...*/) +this.sock.ev.on('call_log' /*, ...*/) +``` + +Requirements: + +* Do not remove event listeners because they appear duplicate. +* Do not reorder event handling without understanding status/webhook effects. +* Do not assume Baileys events are clean, unique, or already normalized. +* Do not add expensive blocking work directly inside socket listeners. + +### 6.2. Main incoming pipeline + +Incoming messages roughly flow as: + +```text +messages.upsert + -> handleIncomingMessagesReceived(...) + -> convertToIncomingWebhooks(...) + -> mapper.webhook.incoming.process.toWebhook(...) + -> mapper.journal.incoming.process.toJournal(...) + -> addToQueueWithRetry(...) for webhook queue + -> repository upsert for history/journal +``` + +Requirements: + +* Preserve the separation between raw socket event, webhook payload, journal payload, queue publication, and history persistence. +* Do not change webhook and journal mappers independently if they must stay aligned. +* Do not remove dedupe/filter behavior without checking duplicate webhooks and history impact. +* Do not assume incoming processing is HTTP delivery. + +### 6.3. Webhook types + +Webhook types include values such as: + +* `incomingMessageReceived` +* `outgoingMessageStatus` +* `outgoingMessageReceived` +* `outgoingAPIMessageReceived` +* `stateInstanceChanged` +* `calls` +* `avatar` + +Requirements: + +* Treat webhook type names as external contracts. +* Do not rename, merge, or split webhook types casually. +* Do not add new webhook types without downstream compatibility planning. +* Preserve payload shape unless intentionally versioning or migrating consumers. + +### 6.4. Webhook delivery + +Webhook delivery is not direct HTTP from socket listeners. + +Current shape: + +```text +WebhookServiceBayles + -> RmqSendService + -> sw-incoming-{id} + -> downstream consumer(s) +``` + +Requirements: + +* Preserve queue-based webhook delivery unless explicitly changing architecture. +* Preserve queue naming conventions. +* Preserve retry publication behavior. +* Do not replace queue publish with direct HTTP delivery as a cleanup. +* Do not block socket event processing on slow downstream webhook consumers. + +### 6.5. Webhook publication retry + +Webhook queue publication uses retry behavior, currently with many attempts and long delay. + +Requirements: + +* Treat retry count/delay as operational behavior. +* Do not reduce retries casually. +* Do not silently drop webhook events. +* If adding failure handling, make it explicit and observable. + +### 6.6. History persistence + +Mongo collections include: + +* `ga_incoming_messages` +* `ga_outgoing_messages` +* `ga_chats` +* `ga_contacts` + +Requirements: + +* Treat collection names and document shapes as externally useful. +* Do not rename fields casually. +* Do not change upsert keys or dedupe semantics without checking history queries. +* Preserve history endpoints behavior when touching persistence. +* Be careful with migrations and backfills. + +### 6.7. History and polling APIs + +Current swcore has history endpoints such as: + +* `GetChatHistory` +* `lastIncomingMessages` +* `lastOutgoingMessages` + +A polling notification API such as `receiveNotification/deleteNotification` was not identified in this service. + +Requirements: + +* Do not invent notification polling behavior in swcore unless explicitly requested. +* Do not present history endpoints as webhook delivery. +* Keep history behavior separate from live event delivery. + +--- + +## 7) Media and downloader behavior + +### 7.1. Media event flow + +Media behavior may branch depending on configuration. + +When downloader integration is enabled: + +```text +media event + -> sw-downloads-{pool} + -> downloader service +``` + +There is also a lazy download endpoint that reads stored message data and uploads/downloads through Media API. + +Requirements: + +* Do not assume all media is downloaded immediately. +* Do not assume downloader service is always enabled. +* Do not bypass Media API integration casually. +* Preserve lazy download behavior unless intentionally changing media architecture. +* Do not load large media into memory without checking runtime impact. + +### 7.2. Media contracts + +Requirements: + +* Preserve media metadata shape in webhook/journal payloads. +* Preserve downloader queue naming. +* Preserve failure behavior when media is unavailable. +* Do not silently mark media downloaded if download/upload failed. +* Do not leak media URLs, tokens, or raw payloads in logs. + +--- + +## 8) Security, secrets, and sensitive data + +### 8.1. REST security boundary + +Many REST routes include `:apiTokenInstance`, but the middleware may be a legacy stub and may not enforce a real token check. + +Requirements: + +* Do not assume REST routes are securely protected by `apiTokenInstance`. +* Do not strengthen or change token validation casually because existing legacy clients may depend on current behavior. +* If asked to improve security, treat it as a contract and migration task, not a tiny middleware patch. +* Be explicit about compatibility impact. + +### 8.2. Sensitive data + +Sensitive data includes: + +* Auth creds +* Sessions +* Prekeys +* Sender keys +* App state sync keys +* Message bodies +* Media URLs +* Phone numbers / contacts +* Webhook payloads +* Redis/Mongo/Rabbit credentials +* SAPI/cache-api tokens +* Instance identifiers when combined with auth material + +Requirements: + +* Do not log sensitive material. +* Do not add debug logs with raw event payloads in production paths. +* Do not expose auth/session material through REST responses. +* Do not include secrets in error messages. +* Do not store secrets in new places without explicit design. + +### 8.3. Existing logging risks + +Known risky areas include: + +* Full event/webhook/journal logging under event logger flags +* Message body logging on send errors +* CacheFactory logging Redis password/user + +Requirements: + +* When touching these areas, reduce sensitive exposure if it is safe and compatible. +* Do not add new broad payload logs. +* Prefer structured, redacted logs. +* Preserve useful operational context without exposing secrets. + +Example preferred style: + +```typescript +this.logger.warn({ + idInstance, + messageId, + errorCode, + temporary: true, +}, 'outgoing command failed temporarily') +``` + +Avoid: + +```typescript +this.logger.warn({ + body: producerMessage, + authData, + redisPassword, +}, 'debug') +``` + +--- + +## 9) Cache, SAPI, and external configuration + +### 9.1. Cache behavior + +The service uses Redis-like cache through `CacheFactory`. + +Cache may affect: + +* Reconnect slots +* Per-instance prefixes +* Prekey TTL +* QR/cache delivery +* Runtime coordination +* SAPI/cache-api settings + +Requirements: + +* Do not treat cache as purely optional decoration. +* Do not change cache key prefixes casually. +* Do not change TTLs without checking runtime behavior. +* Do not log cache credentials. +* Do not replace cache logic with in-memory state unless explicitly designing single-process behavior. + +### 9.2. SAPI/cache-api settings + +Instance settings may come from SAPI/cache-api rather than local database tables. + +Requirements: + +* Do not add local settings as a competing source of truth. +* Do not bypass SAPI updates casually. +* Do not assume local memory has the latest instance configuration. +* Preserve subscription/update behavior at startup. + +--- + +## 10) Public and implicit contracts + +### 10.1. Contract types + +The public and implicit contracts include: + +* REST route paths +* REST request payloads +* REST response payloads +* REST status codes +* REST error behavior +* `apiTokenInstance` route shape +* RabbitMQ queue names +* RabbitMQ message shapes +* RabbitMQ ACK/NACK semantics +* Retry counts and retry delays +* Message ID format and classification +* Webhook event type names +* Webhook payload shape +* Journal/history payload shape +* Mongo collection names +* Mongo document fields +* Auth/session storage categories +* QR websocket payloads +* Status/state enum values +* Downloader queue names +* Media metadata shape +* Log formats consumed by operations +* Timing/order assumptions in consumers + +Requirements: + +* Treat contract changes as serious. +* If a change affects a contract, identify the consumers. +* If consumers are unknown, assume they exist. +* Preserve backward compatibility by default. +* Add compatibility aliases only when they do not create ambiguous behavior. +* Document intentional breaking changes. + +### 10.2. Explicit vs implicit behavior + +Requirements: + +* Do not assume undocumented means unused. +* Do not assume weird means wrong. +* Do not assume duplicate means redundant. +* Do not assume legacy means safe to remove. +* Do not assume a controller path is the only entrypoint. +* Do not assume a queue path has the same semantics as REST. + +--- + +## 11) Code style + +sw-core code should be **careful, explicit, and boring in a good way**. + +### 11.1. General principles + +* Prefer clear procedural flow over abstract indirection. +* Keep runtime paths readable end-to-end. +* Use early returns instead of deep nesting when safe. +* Add small named helpers only when they reduce local risk. +* Avoid speculative abstractions. +* Avoid broad formatting-only diffs. +* Keep stateful behavior visible. +* Preserve operational logs, but redact sensitive values. +* Prefer compatibility over elegance. + +### 11.2. Naming rules + +**Data → nouns:** + +```typescript +idInstance +producerMessage +messageId +payload +webhook +journal +authData +stateAccount +statusInstance +queueName +headers +metadata +``` + +**Actions → verbs:** + +```typescript +spawn +reboot +connect +logout +resetSession +sendMessage +sendFileByUrl +requestPairingCode +changeStateAccount +changeStatusInstance +handleIncomingMessagesReceived +convertToIncomingWebhooks +addToQueueWithRetry +buildOutgoingStatusWebhook +``` + +Requirements: + +* Use names that describe actual behavior. +* Do not use names that imply a cleaner or larger architecture than exists. +* Avoid vague infrastructure poetry such as `orchestrateFlow`, `processMagic`, `unifiedGateway`, `smartDispatcher`. +* Do not rename widely used legacy methods unless explicitly required. + +### 11.3. Function structure pattern + +Preferred local structure: + +1. **Doing some checks** — validate input, instance state, socket availability, required fields +2. **Getting the data** — parse payload, read runtime instance, derive options, fetch config +3. **Defining the functions** — only for small local helpers that clarify the current function +4. **Main flow** — call runtime/socket/mapper/repository/queue +5. **Handling result** — return ID/status/payload or ACK/NACK +6. **Handling errors** — classify temporary/permanent errors, preserve retry semantics + +Example: + +```typescript +async sendMessageFromCommand(producerMessage: ProducerMessage): Promise { + + // Doing some checks + + if (!producerMessage.idInstance) + throw new Error('idInstance is required'); + + + // Getting the data + + const waInstance = await this.throwWaInstance(producerMessage.idInstance); + + + await waInstance.sendMessage( + producerMessage.chatId, + producerMessage.message, + producerMessage.messageID, + ); +} +``` + +### 11.4. Comments + +Comments should explain **why**, not obvious syntax. + +Use these markers consistently when useful: + +```typescript +// Constants +// Variables +// Doing some checks +// Getting the data +// Defining the functions +// FIXME: +// TODO: +// NOTE: +``` + +Good comment: + +```typescript +// NOTE: Keep this check before ACK/NACK handling: temporary socket errors must be retried by RabbitMQ. +``` + +Bad comment: + +```typescript +counter++ // Increment counter by one +``` + +--- + +## 12) Runtime and infrastructure guidance + +### 12.1. Runtime reality + +Requirements: + +* Treat RabbitMQ, Mongo, Redis/cache, SAPI/cache-api, Media API, downloader, and Baileys as part of the working system. +* Do not validate behavior only by reading controller code. +* For socket lifecycle changes, inspect actual spawn/reconnect/auth/event paths. +* For queue changes, inspect producer message shape, consumer setup, ACK/NACK behavior, and downstream effects. +* For persistence changes, inspect query/read paths as well as writes. + +### 12.2. RabbitMQ guidance + +Requirements: + +* Preserve queue names unless explicitly changing topology. +* Preserve `prefetch(1)` unless intentionally changing send concurrency. +* Preserve ACK/NACK semantics. +* Preserve retry behavior for temporary errors. +* Preserve failed webhook behavior on final failure. +* Do not acknowledge failed commands before required failure side effects are completed. +* Do not NACK permanent errors forever. +* Do not publish malformed webhook payloads. + +### 12.3. Mongo guidance + +Requirements: + +* Treat auth collections as protocol-critical. +* Treat history collections as externally useful. +* Do not rename collections or fields without migration. +* Do not change upsert keys casually. +* Do not introduce unbounded writes in high-volume event paths. +* Be careful with indexes and query shapes. +* Do not store raw payloads in new collections without data-retention review. + +### 12.4. Baileys/socket guidance + +Requirements: + +* Do not reimplement messenger protocol behavior locally. +* Do not patch around SDK behavior with HTTP-layer hacks unless explicitly required. +* Do not swallow socket errors without observability. +* Do not change listener registration casually. +* Do not send messages when socket/account state says it is unsafe, unless existing behavior already does so and the task requires preserving it. +* Do not convert runtime state to stateless request/response logic. + +### 12.5. Multi-pod/process caution + +A clear lease/heartbeat ownership model for instances between pods may not exist. + +Requirements: + +* Do not add assumptions that only one pod can own an instance unless verified. +* Do not add local-only coordination for globally visible behavior. +* Be careful with shared cache reconnect slots. +* Be careful with duplicated consumers or duplicated socket sessions. +* If solving multi-pod ownership, treat it as an architecture task. + +--- + +## 13) API growth and change management + +### 13.1. Adding new behavior + +Good fits for this repository: + +* Small runtime-safe helpers around existing Baileys operations +* Explicit additions to existing send/auth/history/media flows +* Mapper extensions that preserve old payloads +* Operational fixes to reconnect/retry behavior +* Security hardening with compatibility planning +* Redacted logging improvements +* Bug fixes with narrow blast radius + +Poor fits unless explicitly required: + +* Full gateway redesign +* New public API product layer +* Generic outbox system +* Approval/workflow/business document logic +* New database ownership model +* New multi-pod lease system hidden inside a small bugfix +* Big FSM rewrite +* Queue topology migration without producer/consumer migration plan + +### 13.2. Backward compatibility + +When changing behavior: + +* Consider whether clients depend on current response shape, status code, error text, message ID, queue payload, webhook event type, or Mongo field. +* Preserve compatibility intentionally. +* Add new fields instead of changing existing fields when possible. +* Do not remove legacy aliases unless explicitly requested. +* If strict validation is added, treat it as a possible breaking change. +* If defaults are changed, explain why. + +### 13.3. Output behavior + +Requirements: + +* Keep webhook payload behavior predictable. +* Keep outgoing status behavior separate from send acceptance. +* Keep history writes aligned with webhook/journal mapping. +* Do not change media output shape casually. +* Do not change state/status event shape casually. +* Do not change ID generation casually. + +--- + +## 14) Domain-specific decision rules + +### 14.1. If the task touches lifecycle + +Read: + +* `wa.instances.service.ts` +* `wa.instance.bayles.client.ts` +* `jobs.ts` +* Auth store integration +* QR gateway if authorization is involved +* Rabbit consumer lifecycle if consumers are involved + +Ask: + +* Does this affect spawn/reboot/connect/logout/reset? +* Does this affect state/status webhooks? +* Does this affect existing sessions? +* Does this affect consumer start/stop? +* Does this affect reconnect behavior? + +Default posture: + +* Make the smallest safe change. +* Preserve existing state names and transitions. +* Avoid centralizing lifecycle logic unless explicitly required. + +### 14.2. If the task touches outbound send + +Read: + +* Controller send endpoint if REST is involved +* `WaInstancesService` send methods +* `WaInstanceBaylesClient` send methods +* RabbitMQ consumer if queue commands are involved +* Webhook status handling if status behavior is involved +* Message ID utility if IDs are involved + +Ask: + +* Is this REST path or RabbitMQ path? +* Who owns `messageID`? +* What happens on temporary error? +* What happens after retry exhaustion? +* Does success mean accepted or delivered? +* Does this affect rate delay or ordering? + +Default posture: + +* Preserve accepted-vs-delivered semantics. +* Preserve ACK/NACK behavior. +* Preserve per-instance serialization. + +### 14.3. If the task touches incoming events or webhooks + +Read: + +* Socket listener registration +* `WebhookServiceBayles` +* Mapper interfaces +* Webhook mapper tree +* Journal mapper tree +* RabbitMQ publisher +* History repositories/entities + +Ask: + +* Which raw event starts the flow? +* Which webhook type is emitted? +* Is history written? +* Is media involved? +* Is this incoming, outgoing, status, call, avatar, or state? +* Does dedupe/filter behavior change? + +Default posture: + +* Preserve payload shape. +* Preserve queue publication. +* Preserve history alignment. + +### 14.4. If the task touches auth/session storage + +Read: + +* `use-multi-mongo-auth-state.ts` +* `auth.data.mongo.service.ts` +* Auth models +* Baileys socket creation path +* Reboot/spawn path + +Ask: + +* Does this affect existing sessions? +* Does this affect key categories? +* Does this affect save timing? +* Does this affect serialization? +* Does this leak sensitive material? +* Is migration required? + +Default posture: + +* Treat as high-risk. +* Avoid changes unless necessary. +* Never log raw auth data. + +### 14.5. If the task touches security + +Read: + +* Middleware +* Controller route structure +* Instance account model +* Auth/session storage +* Logging paths +* Config/env usage + +Ask: + +* Is this changing a legacy behavior? +* Are existing clients relying on weak/no validation? +* Is rollout/migration needed? +* Are secrets or message bodies exposed? +* Can hardening be done with compatibility? + +Default posture: + +* Improve redaction safely. +* Do not silently break clients. +* Do not overstate current security guarantees. + +### 14.6. If the task touches media + +Read: + +* Webhook media handling +* Downloader service +* Lazy download endpoint +* Media API client +* Stored message entities + +Ask: + +* Is downloader enabled? +* Is this lazy or eager download? +* Is metadata shape changing? +* Are files or URLs sensitive? +* Does this affect memory usage? + +Default posture: + +* Preserve branch behavior. +* Avoid loading large files in memory. +* Preserve Media API contract. + +--- + +## 15) Documentation and verification + +### 15.1. Documentation responsibilities + +If functionality changes, update as needed: + +* Route/API docs +* Queue contract docs +* Webhook payload examples +* State/status documentation +* Auth/session migration notes +* Environment/config notes +* Operational runbooks +* Comments near public behavior +* Examples used by downstream services + +Requirements: + +* Do not let code and docs diverge further. +* If docs are already incomplete, add narrowly useful notes near the changed behavior. +* Do not document guessed behavior as certain. + +### 15.2. Verification priorities + +Because this repository is a runtime integration layer, useful checks are usually flow-based. + +Prefer: + +* Real or integration-style send path checks +* RabbitMQ command handler checks +* ACK/NACK behavior checks +* Webhook payload shape checks +* History persistence checks +* Auth storage compatibility checks +* QR/pairing flow checks +* Reconnect/reboot smoke checks +* Media downloader/lazy download checks when touched + +Less useful by itself: + +* Decorative unit tests around mocked helpers that do not exercise runtime semantics +* Snapshot tests that bless accidental legacy behavior without explanation +* Controller-only checks for socket/event behavior + +### 15.3. Change summary requirements + +When summarizing a change, include: + +* What flow was changed +* Which contracts were preserved +* Whether REST, RabbitMQ, webhook, Mongo, auth, or media behavior changed +* Whether retries/ACK/NACK/delays were changed +* Whether message ID or state/status semantics changed +* What was verified +* What remains risky or unverified + +Example summary: + +```text +Changed the RabbitMQ outbound SendFileByUrl handling to classify uploader timeout as a temporary error. + +Preserved: +- sw-outgoing-{id} queue contract +- producerMessage.messageID usage +- per-instance send mutex +- ACK after final failed webhook +- REST send behavior + +Verified: +- temporary timeout returns NACK +- permanent validation error is not retried +- failed webhook is emitted after retry exhaustion +``` + +--- + +## 16) Do Not + +* ❌ Don’t describe this repository as a clean standalone REST API product. +* ❌ Don’t treat REST controllers as the core of the system. +* ❌ Don’t ignore `WaInstancesService` and `WaInstanceBaylesClient` when touching runtime behavior. +* ❌ Don’t change live socket lifecycle casually. +* ❌ Don’t change auth/session storage shape casually. +* ❌ Don’t log auth material, session keys, prekeys, sender keys, message bodies, media URLs, or credentials. +* ❌ Don’t assume `apiTokenInstance` is a strong security boundary. +* ❌ Don’t silently strengthen legacy REST auth without a compatibility plan. +* ❌ Don’t merge `StateAccount` and `StatusInstance`. +* ❌ Don’t add, rename, or remove state/status enum values casually. +* ❌ Don’t introduce a new FSM as a cleanup. +* ❌ Don’t change QR or pairing behavior without tracing the full auth flow. +* ❌ Don’t convert REST send into an async outbox API unless explicitly required. +* ❌ Don’t treat send success as delivered/read success. +* ❌ Don’t change message ID generation or parsing casually. +* ❌ Don’t rename RabbitMQ queues casually. +* ❌ Don’t change `ProducerMessage` shape casually. +* ❌ Don’t remove `prefetch(1)` casually. +* ❌ Don’t bypass ACK/NACK retry semantics. +* ❌ Don’t remove per-instance send mutexes casually. +* ❌ Don’t remove send delays/rate throttling as “cleanup”. +* ❌ Don’t replace queue-based webhook delivery with direct HTTP delivery casually. +* ❌ Don’t change webhook type names casually. +* ❌ Don’t reshape webhook payloads without downstream compatibility planning. +* ❌ Don’t change journal/history mapping independently from webhook behavior when they must stay aligned. +* ❌ Don’t rename Mongo collections or fields without migration. +* ❌ Don’t change auth key category routing without understanding Baileys expectations. +* ❌ Don’t add broad abstractions that hide runtime flow. +* ❌ Don’t migrate frameworks or rewrite services for style. +* ❌ Don’t perform formatting sweeps mixed with behavior changes. +* ❌ Don’t remove legacy behavior just because it looks strange. +* ❌ Don’t assume undocumented behavior is unused. +* ❌ Don’t assume duplicate-looking logic is redundant. +* ❌ Don’t add silent fallbacks that hide protocol/runtime failures. +* ❌ Don’t add local-only state for globally visible multi-pod behavior. +* ❌ Don’t bypass SAPI/cache-api as source of instance settings. +* ❌ Don’t treat Redis/cache as a harmless optimization. +* ❌ Don’t add downloader/media behavior that loads large files into memory without need. +* ❌ Don’t expose raw errors from Baileys, Mongo, RabbitMQ, Redis, or Media API if they contain sensitive data. +* ❌ Don’t deepen divergence between implementation, docs, queues, webhook examples, and operational reality. +* ❌ Don’t turn this legacy runtime-worker into architecture theater. + +--- + +## 17) Pre-change checklist + +* [ ] The affected flow was identified: lifecycle, auth, outbound, inbound, webhook, history, media, security, cache, or runtime setup. +* [ ] The real runtime path was traced end-to-end. +* [ ] Legacy contracts were considered before changing behavior. +* [ ] Public and implicit contracts were preserved unless intentionally changed. +* [ ] Queue names and RabbitMQ ACK/NACK behavior were preserved unless intentionally changed. +* [ ] Message ID semantics were preserved unless intentionally changed. +* [ ] Send accepted vs delivered/read semantics were not confused. +* [ ] StateAccount and StatusInstance semantics were preserved. +* [ ] Auth/session storage shape and save behavior were not broken. +* [ ] Sensitive data was not logged or exposed. +* [ ] Webhook payload shape and event types were preserved unless intentionally changed. +* [ ] Journal/history persistence stayed aligned with webhook behavior. +* [ ] Cache/SAPI/runtime assumptions were checked. +* [ ] Minimal sufficient diff was used. +* [ ] No broad refactors, formatting sweeps, or speculative abstractions were introduced. +* [ ] Documentation/examples/config notes were updated if behavior changed. +* [ ] Verification matched the real integration flow, not only isolated helper code. +* [ ] Remaining risk or uncertainty was stated clearly. diff --git a/README.md b/README.md index 63340ab..3d519a1 100644 --- a/README.md +++ b/README.md @@ -517,6 +517,141 @@ def show_interactive_buttons_reply_handler(notification: Notification) -> None: bot.run_forever() ``` +## Optional voice calls (VoIP) + +The `whatsapp_chatbot_python.calls` package executes one outgoing WhatsApp +voice call through the GreenAPI SDK, with an OpenAI Realtime conversation, +timeouts, audio playback and a best-effort MP3 recording. It requires **Python +3.11+**, an authorized VoIP-capable instance and an OpenAI API key. Install the +optional dependencies (or use `'.[voip]'` when installing from this checkout): + +```shell +python -m pip install 'whatsapp-chatbot-python[voip]' +``` + +The extra pins the SDK and audio/provider versions used by the source demo. +The ordinary chatbot import does not load OpenAI, aiortc or PyAV. Call models, +contracts and the FSM are available from `whatsapp_chatbot_python.calls`; +import the executor explicitly from `whatsapp_chatbot_python.calls.service`. + +### Execute one call + +The model and voice are supplied by the application. This example performs an +actual call when run with valid credentials: + +```python +import asyncio +import logging +import os + +from whatsapp_chatbot_python.calls import ( + CallEvent, CallSession, CallStateMachine, +) +from whatsapp_chatbot_python.calls.service import WhatsAppCallService + +service = WhatsAppCallService( + api_url="https://api.green-api.com", + id_instance=os.environ["ID_INSTANCE"], + api_token_instance=os.environ["API_TOKEN_INSTANCE"], + openai_api_key=os.environ["OPENAI_API_KEY"], + realtime_model=os.environ["REALTIME_MODEL"], + realtime_voice=os.environ["REALTIME_VOICE"], + ring_timeout_seconds=30, + talk_timeout_seconds=120, + shutdown_timeout_seconds=10, + logger=logging.getLogger("calls"), +) + +async def call_once(): + session = CallSession( + sender_id="79123456789@c.us", + chat_id="79123456789@c.us", + language="ru", + ) + fsm = CallStateMachine() + fsm.apply(session, CallEvent.ENQUEUED) + fsm.apply(session, CallEvent.DEQUEUED) + result = await service.execute(session, fsm.apply) + try: + print(session.state, session.end_reason, session.error_code) + if result.recording_path is not None: + print("MP3:", result.recording_path) + # Upload or copy the recording here, before removing it. + finally: + if result.recording_path is not None: + result.recording_path.unlink(missing_ok=True) + +asyncio.run(call_once()) +``` + +`execute(session, transition)` retains the demo's interface. Prepare the session +in `DIALING` (the `ENQUEUED` / `DEQUEUED` transitions above do this). The callback +must **synchronously apply each transition to the same session**, not merely +log it. A coordinator should apply transitions under its existing lock. +The final state, end reason and error code are on the mutated `CallSession`; +`CallExecutionResult` contains only `recording_path: Path | None`. + +Run calls serially on each service instance. `request_stop()` is thread-safe +and permanently requests shutdown of that executor; create a new executor to +resume. Initialization failures may propagate to the caller, as in the demo; +the application/coordinator must handle them. Once execution reaches its main +call loop, handled errors are reflected through FSM transitions. + +### Integrate with an existing demo coordinator + +Replace the old service import with: + +```python +from whatsapp_chatbot_python.calls.service import WhatsAppCallService +from whatsapp_chatbot_python.calls.contracts import CallExecutor, TransitionCallback +from whatsapp_chatbot_python.calls import CallSession, CallStateMachine +``` + +Keep the constructor arguments and the existing worker invocation: + +```python +result = asyncio.run(executor.execute(session, coordinator.transition)) +``` + +Queueing, duplicate prevention, chat commands, menu state, localized result +texts and uploading the recording remain in the application. Keep the call in +the coordinator's worker: waiting for it inside a synchronous message handler +blocks notification processing. The executor creates its own GreenAPI client; +no `Notification` or chatbot API object needs to cross threads. + +Old model/FSM imports can be retained with explicit re-exports from this package. +Keep `EnqueueResult` in the demo. Move test mock targets from +`internal.calls.service.*` to `whatsapp_chatbot_python.calls.service.*` (and +likewise for audio modules): re-exporting a class does not redirect its globals. + +### Audio and recording behavior + +The original audio pipeline is preserved: mono PCM16 at 24 kHz, 20 ms output +frames, caller interruption, bounded playback buffering, and replacement audio +tracks on reconnect while retaining one conversation and greeting. Greeting +starts after remote acceptance and audio bridge negotiation; negotiation does +not itself guarantee live ICE/DTLS audio. The talk timer starts at remote +acceptance, including time waiting for the bridge. + +Recording mixes both parties and keeps pauses. A recording is returned only +for `REMOTE_ENDED` or `TALK_TIMEOUT`, and may still be `None` if there was no +audio or recording failed. Failed-call partial recordings are discarded. +The returned temporary MP3 belongs to the caller: upload/copy it if needed and +always delete it afterwards. Recording failure does not by itself fail the call. + +### Tests + +```shell +python -m pip install -e '.[voip]' pytest +python -m pytest tests +``` + +The call tests cover dialing, acceptance/bridge ordering, reconnects, shutdown, +timeouts, playback interruption and real MP3 encode/decode. Network/provider +behavior is simulated. Without optional audio dependencies, pytest collects +only the lightweight FSM/runtime tests; on Python below 3.11 it omits the call +suite. CI installs the extra and runs the full suite on Python 3.11–3.13. + ## Service methods documentation [Service methods documentation](https://green-api.com/en/docs/api/) diff --git a/docs/README.md b/docs/README.md index 3d46e91..9f491f8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -513,6 +513,140 @@ def show_interactive_buttons_reply_handler(notification: Notification) -> None: bot.run_forever() ``` +## Голосовые звонки (необязательное дополнение VoIP) + +Пакет `whatsapp_chatbot_python.calls` выполняет один исходящий WhatsApp-звонок: +подключение через GreenAPI SDK, голосовой разговор с OpenAI Realtime, +тайм-ауты, воспроизведение и запись MP3. Требуются **Python 3.11+**, +авторизованный инстанс с поддержкой VoIP и ключ OpenAI. + +```shell +python -m pip install 'whatsapp-chatbot-python[voip]' +``` + +Для установки из исходников используйте `python -m pip install -e '.[voip]'`. +Дополнение фиксирует версии SDK, OpenAI и аудиобиблиотек из исходного демо. +Обычный импорт чат-бота не загружает OpenAI, aiortc или PyAV. + +### Выполнить один звонок + +Модель и голос задаёт приложение. При запуске с действительными реквизитами +этот пример совершит настоящий звонок: + +```python +import asyncio +import logging +import os + +from whatsapp_chatbot_python.calls import ( + CallEvent, CallSession, CallStateMachine, +) +from whatsapp_chatbot_python.calls.service import WhatsAppCallService + +service = WhatsAppCallService( + api_url="https://api.green-api.com", + id_instance=os.environ["ID_INSTANCE"], + api_token_instance=os.environ["API_TOKEN_INSTANCE"], + openai_api_key=os.environ["OPENAI_API_KEY"], + realtime_model=os.environ["REALTIME_MODEL"], + realtime_voice=os.environ["REALTIME_VOICE"], + ring_timeout_seconds=30, + talk_timeout_seconds=120, + shutdown_timeout_seconds=10, + logger=logging.getLogger("calls"), +) + +async def call_once(): + session = CallSession( + sender_id="79123456789@c.us", + chat_id="79123456789@c.us", + language="ru", + ) + fsm = CallStateMachine() + fsm.apply(session, CallEvent.ENQUEUED) + fsm.apply(session, CallEvent.DEQUEUED) + result = await service.execute(session, fsm.apply) + try: + print(session.state, session.end_reason, session.error_code) + if result.recording_path is not None: + print("MP3:", result.recording_path) + # Upload or copy the recording here, before removing it. + finally: + if result.recording_path is not None: + result.recording_path.unlink(missing_ok=True) + +asyncio.run(call_once()) +``` + +Сохранён интерфейс `await service.execute(session, transition)`. +Перед выполнением сессия должна находиться в `DIALING`: выше её подготавливают +переходы `ENQUEUED` и `DEQUEUED`. Callback обязан синхронно применять FSM к +той же сессии. В координаторе это делается под его существующей блокировкой. +Callback, который только логирует событие, для этого контракта не подходит. + +Итоговое состояние, причина завершения и код ошибки находятся в изменённой +`CallSession`. `CallExecutionResult` возвращает только путь к записи либо `None`. +На одном исполнителе звонки выполняются последовательно. +`request_stop()` можно вызвать из другого потока: он навсегда запрашивает +остановку данного исполнителя. Для возобновления создайте новый объект. +Ошибки начальной подготовки могут выбрасываться вызывающей стороне; координатор +должен их обрабатывать, как в исходном демо. + +### Подключение к демо + +Замените импорт сервиса и общих контрактов: + +```python +from whatsapp_chatbot_python.calls.service import WhatsAppCallService +from whatsapp_chatbot_python.calls.contracts import CallExecutor, TransitionCallback +from whatsapp_chatbot_python.calls import CallSession, CallStateMachine +``` + +Аргументы конструктора сохраняются. В рабочем потоке координатора остаётся: + +```python +result = asyncio.run(executor.execute(session, coordinator.transition)) +``` + +Очередь, защита от повторных заявок, команды чата, меню, локализованные ответы +и загрузка записи остаются в приложении. Не ждите звонок внутри синхронного +обработчика сообщений: это задержит обработку остальных уведомлений. +Исполнитель создаёт собственный GreenAPI-клиент; передавать ему `Notification` +или API-клиент обработчика не требуется. + +Старые импорты моделей и FSM можно сохранить явными реэкспортами. +`EnqueueResult` остаётся в демо. Пути подмен в тестах нужно изменить с +`internal.calls.service.*` на `whatsapp_chatbot_python.calls.service.*`, +аналогично для аудиомодулей: реэкспорт класса не меняет его глобальные зависимости. + +### Аудио и запись + +Сохранены PCM16 mono 24 кГц, кадры по 20 мс, прерывание ответа собеседником, +ограничение буфера и замена аудиотреков при переподключении без нового разговора +и повторного приветствия. Приветствие начинается после ответа и согласования +моста; согласование само по себе не подтверждает наличие живого аудио ICE/DTLS. +Таймер разговора запускается при ответе собеседника, включая ожидание моста. + +Запись объединяет обе стороны и сохраняет паузы. Файл возвращается только при +`REMOTE_ENDED` и `TALK_TIMEOUT`; отсутствие аудио или ошибка записи могут дать +`None` и в этих случаях. Частичные записи неуспешных звонков удаляются. +Возвращённым временным MP3 владеет приложение: после отправки или копирования +оно обязано удалить файл, предпочтительно через `finally`. +Ошибка записи сама по себе не завершает разговор как неуспешный. + +### Проверка + +```shell +python -m pip install -e '.[voip]' pytest +python -m pytest tests +``` + +Тесты проверяют дозвон, порядок ответа и готовности моста, переподключения, +остановку, тайм-ауты, прерывания аудио и настоящее кодирование/декодирование MP3. +Сеть и провайдер моделируются. Без аудиозависимостей pytest собирает только +лёгкие тесты FSM и таймеров; на Python ниже 3.11 тесты звонков не собираются. +CI устанавливает дополнение и запускает полный набор на Python 3.11–3.13. + ## Документация по методам сервиса [Документация по методам сервиса](https://green-api.com/docs/api/) diff --git a/setup.py b/setup.py index a46df64..d99b6ed 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ author="GREEN API", author_email="support@green-api.com", url="https://github.com/green-api/whatsapp-chatbot-python", - packages=find_packages(exclude=["tests"]), + packages=find_packages(include=["whatsapp_chatbot_python", "whatsapp_chatbot_python.*"]), classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -44,5 +44,13 @@ " (CC BY-ND 4.0)" ), install_requires=["whatsapp-api-client-python>=0.0.53"], + extras_require={ + "voip": [ + "whatsapp-api-client-python[voip]==0.0.55", + "openai==3.20.0", + "aiortc==1.15.0", + "av==17.1.0", + ], + }, python_requires=">=3.7" ) diff --git a/tests/calls/__init__.py b/tests/calls/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/calls/conftest.py b/tests/calls/conftest.py new file mode 100644 index 0000000..492217b --- /dev/null +++ b/tests/calls/conftest.py @@ -0,0 +1,12 @@ +"""Keep base installations testable without installing the optional audio stack.""" + +import sys +from importlib.util import find_spec + +collect_ignore_glob = [] +if sys.version_info < (3, 11): + collect_ignore_glob = ["test_*.py"] +elif any(find_spec(name) is None for name in ("openai", "aiortc", "av")): + collect_ignore_glob = [ + "test_call_service.py", "test_realtime_voice.py", "test_call_recording.py" + ] diff --git a/tests/calls/test_call_recording.py b/tests/calls/test_call_recording.py new file mode 100644 index 0000000..2a589c8 --- /dev/null +++ b/tests/calls/test_call_recording.py @@ -0,0 +1,77 @@ +from array import array +from unittest.mock import patch +from queue import Full +from whatsapp_chatbot_python.calls.recording import CallRecorder, RATE +import logging +import unittest +import av + + +def pcm(value: int, samples: int) -> bytes: + return array("h", [value] * samples).tobytes() + + +class CallRecordingTest(unittest.TestCase): + def test_mixes_both_sides_and_keeps_pause_and_reconnect(self) -> None: + clock = [0.0] + recorder = CallRecorder(logging.getLogger("recording-test")) + + with patch("whatsapp_chatbot_python.calls.recording.monotonic", side_effect=lambda: clock[0]): + recorder.set_generation(1) + recorder.start() + + clock[0] = 0.1 + + recorder.add_caller(pcm(4000, RATE // 10), 1) # capture starts at zero + + clock[0] = 0.0 + + recorder.add_bot(pcm(3000, RATE // 10), 1) + recorder.set_generation(2) + + clock[0] = 0.2 + + recorder.add_bot(pcm(10000, RATE // 10), 1) # old bridge ignored + recorder.add_bot(pcm(5000, RATE // 10), 2) + + path = recorder.finish() + + self.assertIsNotNone(path) + + try: + with av.open(str(path)) as container: + frames = [frame for packet in container.demux(audio=0) for frame in packet.decode()] + + converted = av.AudioResampler(format="s16", layout="mono", rate=RATE) + samples = array("h") + + for frame in frames: + for output in converted.resample(frame): + samples.frombytes(bytes(output.planes[0])[:output.samples * 2]) + + # MP3 is lossy; check broad levels in each time segment. + def level(start, end): + section = samples[int(start * RATE):int(end * RATE)] + return sum(section) / len(section) + + self.assertGreater(level(0.03, 0.07), 5500) + self.assertLess(abs(level(0.13, 0.17)), 500) + self.assertGreater(level(0.23, 0.27), 3500) + self.assertLess(level(0.23, 0.27), 8000) + finally: + path.unlink(missing_ok=True) + + def test_no_audio_or_queue_overflow_disables_recording(self) -> None: + recorder = CallRecorder(logging.getLogger("recording-test")) + + self.assertIsNone(recorder.finish()) + + recorder = CallRecorder(logging.getLogger("recording-test")) + + recorder.set_generation(1) + recorder.start() + + with patch.object(recorder._queue, "put_nowait", side_effect=Full): + recorder.add_bot(pcm(1000, 480), 1) + + self.assertIsNone(recorder.finish()) diff --git a/tests/calls/test_call_runtime.py b/tests/calls/test_call_runtime.py new file mode 100644 index 0000000..2ea739d --- /dev/null +++ b/tests/calls/test_call_runtime.py @@ -0,0 +1,23 @@ +from whatsapp_chatbot_python.calls.models import CallState +from whatsapp_chatbot_python.calls.runtime import CallRuntime, RuntimeEvent, RuntimeEventType +import unittest + + +class CallRuntimeTest(unittest.IsolatedAsyncioTestCase): + async def test_queued_event_wins_over_expired_timer(self) -> None: + runtime = CallRuntime( + ring_timeout_seconds=0, + talk_timeout_seconds=1, + bridge_timeout_seconds=1, + ) + runtime.start_ringing() + runtime.events.put_nowait(RuntimeEvent.bridge_ready()) + + event = await runtime.next_event(CallState.RINGING) + + self.assertIsNotNone(event) + self.assertEqual(event.type, RuntimeEventType.BRIDGE_READY) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/calls/test_call_service.py b/tests/calls/test_call_service.py new file mode 100644 index 0000000..79a16f0 --- /dev/null +++ b/tests/calls/test_call_service.py @@ -0,0 +1,427 @@ +from pathlib import Path +from tempfile import NamedTemporaryFile +from threading import Thread +from unittest.mock import patch +import asyncio +import logging +import unittest + +from whatsapp_chatbot_python.calls.models import CallEndReason, CallEvent, CallSession, CallState +from whatsapp_chatbot_python.calls.service import WhatsAppCallService +from whatsapp_chatbot_python.calls.state_machine import CallStateMachine + + +class FakeVoice: + instances = [] + fail_start = False + fail_greeting = False + + def __init__(self, **options): + self.options = options + self.greetings = 0 + self.closed = False + self.instances.append(self) + + async def start(self): + if self.fail_start: + raise RuntimeError("OpenAI unavailable") + + async def close(self): + self.closed = True + + async def greet_once(self): + if self.greetings: + return + + self.greetings += 1 + + if self.fail_greeting: + self.options["on_error"](RuntimeError("OpenAI failed")) + + async def new_output_track(self): + self.tracks = getattr(self, "tracks", []) + track = FakeTrack() + + self.tracks.append(track) + + return track + + def new_input_sink(self): + self.sinks = getattr(self, "sinks", []) + sink = FakeSink() + + self.sinks.append(sink) + + return sink + + +class FakeTrack: + def __init__(self): + self.stopped = False + + def stop(self): + self.stopped = True + + +class FakeSink: + def __init__(self): + self.closed = False + + async def attach(self, track): + pass + + async def close(self): + self.closed = True + + +class FakeCalls: + def __init__(self, client): + self.client = client + self.listeners = {} + self.closed = False + self.audio_factory = None + self.devices = [] + + def on(self, name, callback): + self.listeners.setdefault(name, []).append(callback) + + def emit(self, name, detail=None): + for callback in self.listeners.get(name, []): + callback(detail) + + async def openAsync(self, *, timeout): + self.client.actions.append("open") + + if self.client.open_error: + raise RuntimeError("callsRtc connection refused") + + asyncio.get_running_loop().call_soon( + self.emit, "state", CallStateDetail(self.client.initial_state) + ) + + async def startAudioAsync(self): + self.client.actions.append("bridge") + self.devices.append(await self.audio_factory()) + + if self.client.bridge_error: + raise RuntimeError("audio bridge failed") + + if self.client.answer and not self.client.answer_after_bridge: + self.emit("state", CallStateDetail("on-call")) + + if self.client.answer and self.client.answer_after_bridge: + asyncio.get_running_loop().call_later( + 0.001, self.emit, "state", + CallStateDetail("on-call"), + ) + + if self.client.reconnect: + self.emit("disconnect", {"permanent": False}) + await self.devices[-1].close() + self.devices.append(await self.audio_factory()) + + if self.client.remote_hangup: + asyncio.get_running_loop().call_later( + 0.01, self.emit, "state", + CallStateDetail("idle", "hangup"), + ) + + async def closeAsync(self): + if self.client.close_hangs: + await asyncio.Event().wait() + + for audio in self.devices: + await audio.close() + + self.closed = True + + +class CallStateDetail: + def __init__(self, state, reason=None): + self.state = state + self.reason = reason + + +class FakeClient: + initial_state = "idle" + answer = True + remote_hangup = True + bridge_error = False + open_error = False + reconnect = False + answer_after_bridge = False + close_hangs = False + instances = [] + + def __init__(self, id_instance, api_token_instance, *, host): + self.voip = self + self.calls = FakeCalls(self) + self.actions = [] + self.instances.append(self) + + def connect(self, *, audio_factory): + self.calls.audio_factory = audio_factory + return self.calls + + async def dialAsync(self, target): + self.actions.append(("dial", target)) + + async def hangUpAsync(self): + self.actions.append("hangup") + + +class CallServiceTest(unittest.IsolatedAsyncioTestCase): + def setUp(self): + for name, value in { + "initial_state": "idle", + "answer": True, + "remote_hangup": True, + "bridge_error": False, + "open_error": False, + "reconnect": False, + "answer_after_bridge": False, + "close_hangs": False, + }.items(): + setattr(FakeClient, name, value) + + FakeClient.instances = [] + FakeVoice.instances = [] + FakeVoice.fail_start = False + FakeVoice.fail_greeting = False + + for target, fake in ( + ("whatsapp_chatbot_python.calls.service.GreenAPI", FakeClient), + ("whatsapp_chatbot_python.calls.service.VoiceBotSession", FakeVoice), + ): + patcher = patch(target, fake) + patcher.start() + self.addCleanup(patcher.stop) + + def make_session(self): + fsm = CallStateMachine() + session = CallSession("sender@c.us", "sender@c.us", "ru") + + fsm.apply(session, CallEvent.ENQUEUED) + fsm.apply(session, CallEvent.DEQUEUED) + + return session, fsm.apply + + def make_service(self, ring=1, talk=1, shutdown=10): + logger = logging.getLogger("call-service-test") + logger.disabled = True + return WhatsAppCallService( + api_url="https://example.invalid", id_instance="1", + api_token_instance="token", openai_api_key="test-key", + realtime_model="gpt-realtime-2.1", realtime_voice="marin", + ring_timeout_seconds=ring, talk_timeout_seconds=talk, + shutdown_timeout_seconds=shutdown, logger=logger, + ) + + async def test_dial_voice_bridge_and_remote_hangup(self): + session, transition = self.make_session() + + await self.make_service().execute(session, transition) + + client = FakeClient.instances[0] + voice = FakeVoice.instances[0] + + self.assertEqual(client.actions, ["open", ("dial", "sender@c.us"), "bridge"]) + self.assertEqual(session.state, CallState.REMOTE_ENDED) + self.assertEqual(session.end_reason, CallEndReason.REMOTE_HANGUP) + self.assertEqual(voice.greetings, 1) + self.assertTrue(voice.closed) + self.assertTrue(client.calls.closed) + self.assertTrue(voice.tracks[0].stopped) + self.assertTrue(voice.sinks[0].closed) + + async def test_reconnect_keeps_one_conversation_and_greeting(self): + FakeClient.reconnect = True + session, transition = self.make_session() + + await self.make_service().execute(session, transition) + self.assertEqual(len(FakeVoice.instances), 1) + self.assertEqual(FakeVoice.instances[0].greetings, 1) + self.assertEqual(len(FakeClient.instances[0].calls.devices), 2) + self.assertEqual(len(FakeVoice.instances[0].tracks), 2) + self.assertTrue(all(track.stopped for track in FakeVoice.instances[0].tracks)) + self.assertTrue(all(sink.closed for sink in FakeVoice.instances[0].sinks)) + + async def test_bridge_before_answer_greets_only_after_answer(self): + FakeClient.answer_after_bridge = True + session, transition = self.make_session() + + await self.make_service().execute(session, transition) + self.assertEqual(session.state, CallState.REMOTE_ENDED) + self.assertEqual(FakeVoice.instances[0].greetings, 1) + + async def test_voice_failure_hangs_up_once(self): + FakeClient.remote_hangup = False + FakeVoice.fail_greeting = True + session, transition = self.make_session() + + await self.make_service().execute(session, transition) + self.assertEqual(session.state, CallState.FAILED) + self.assertEqual(session.end_reason, CallEndReason.ERROR) + self.assertEqual(FakeClient.instances[0].actions.count("hangup"), 1) + + async def test_openai_start_failure_does_not_dial(self): + FakeVoice.fail_start = True + session, transition = self.make_session() + + with self.assertRaises(RuntimeError): + await self.make_service().execute(session, transition) + + self.assertEqual(FakeClient.instances, []) + self.assertTrue(FakeVoice.instances[0].closed) + + async def test_ring_and_talk_timeout_hang_up(self): + for answer, expected in ((False, CallState.RING_TIMEOUT), (True, CallState.TALK_TIMEOUT)): + with self.subTest(expected=expected): + FakeClient.answer = answer + FakeClient.remote_hangup = False + session, transition = self.make_session() + + await self.make_service(ring=0.01, talk=0.01).execute(session, transition) + self.assertEqual(session.state, expected) + self.assertEqual(FakeClient.instances[-1].actions.count("hangup"), 1) + + async def test_busy_instance_is_not_dialed(self): + FakeClient.initial_state = "on-call" + session, transition = self.make_session() + + await self.make_service().execute(session, transition) + self.assertEqual(session.state, CallState.FAILED) + self.assertEqual(FakeClient.instances[0].actions, ["open"]) + + async def test_socket_open_failure_does_not_dial(self): + FakeClient.open_error = True + session, transition = self.make_session() + + await self.make_service().execute(session, transition) + + self.assertEqual(session.state, CallState.FAILED) + self.assertEqual(FakeClient.instances[0].actions, ["open"]) + self.assertTrue(FakeClient.instances[0].calls.closed) + + async def test_hung_connection_close_has_deadline(self): + FakeClient.close_hangs = True + session, transition = self.make_session() + + await asyncio.wait_for(self.make_service(shutdown=0.01).execute(session, transition), timeout=1) + self.assertEqual(session.state, CallState.REMOTE_ENDED) + + async def test_bridge_failure_hangs_up(self): + FakeClient.bridge_error = True + session, transition = self.make_session() + + await self.make_service().execute(session, transition) + self.assertEqual(session.state, CallState.FAILED) + self.assertEqual(FakeClient.instances[0].actions.count("hangup"), 1) + + async def test_shutdown_hangs_up(self): + FakeClient.answer = False + FakeClient.remote_hangup = False + session, transition = self.make_session() + + service = self.make_service() + task = asyncio.create_task(service.execute(session, transition)) + + while session.state != CallState.RINGING: + await asyncio.sleep(0) + + service.request_stop() + + await task + + self.assertEqual(session.end_reason, CallEndReason.SHUTDOWN) + self.assertEqual(FakeClient.instances[0].actions.count("hangup"), 1) + + async def test_shutdown_from_another_thread_hangs_up(self): + FakeClient.answer = False + FakeClient.remote_hangup = False + session, transition = self.make_session() + service = self.make_service() + task = asyncio.create_task(service.execute(session, transition)) + + async def wait_for_ringing(): + while session.state != CallState.RINGING: + if task.done(): + await task + self.fail(f"Call finished before ringing: {session.state}") + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_ringing(), timeout=1) + + thread = Thread(target=service.request_stop) + thread.start() + thread.join(timeout=1) + self.assertFalse(thread.is_alive()) + await asyncio.wait_for(task, timeout=1) + + self.assertEqual(session.end_reason, CallEndReason.SHUTDOWN) + self.assertEqual(FakeClient.instances[0].actions.count("hangup"), 1) + + async def test_stopped_executor_does_not_start_resources(self): + service = self.make_service() + service.request_stop() + for _ in range(2): + session, transition = self.make_session() + result = await service.execute(session, transition) + self.assertEqual(session.end_reason, CallEndReason.SHUTDOWN) + self.assertIsNone(result.recording_path) + self.assertEqual(FakeClient.instances, []) + self.assertEqual(FakeVoice.instances, []) + + async def test_executor_can_run_successive_calls(self): + service = self.make_service() + for _ in range(2): + session, transition = self.make_session() + await service.execute(session, transition) + self.assertEqual(session.state, CallState.REMOTE_ENDED) + self.assertEqual(len(FakeClient.instances), 2) + self.assertEqual(len(FakeVoice.instances), 2) + self.assertTrue(all(voice.closed for voice in FakeVoice.instances)) + self.assertIsNone(service._active_loop) + self.assertIsNone(service._active_runtime) + + async def test_failed_conversation_discards_partial_recording(self): + FakeClient.remote_hangup = False + FakeVoice.fail_greeting = True + session, transition = self.make_session() + + with NamedTemporaryFile(suffix=".mp3", delete=False) as file: + path = Path(file.name) + + self.addCleanup(path.unlink, missing_ok=True) + + class FakeRecorder: + def __init__(self, logger): + pass + + def start(self): + pass + + def finish(self): + return path + + with patch("whatsapp_chatbot_python.calls.service.CallRecorder", FakeRecorder): + result = await self.make_service().execute(session, transition) + + self.assertEqual(session.end_reason, CallEndReason.ERROR) + self.assertIsNone(result.recording_path) + self.assertFalse(path.exists()) + + async def test_stop_before_execute_does_not_start_a_call(self): + session, transition = self.make_session() + + service = self.make_service() + + service.request_stop() + + result = await service.execute(session, transition) + + self.assertEqual(session.end_reason, CallEndReason.SHUTDOWN) + self.assertEqual(session.state, CallState.FAILED) + self.assertIsNone(result.recording_path) + self.assertEqual(FakeClient.instances, []) + self.assertEqual(FakeVoice.instances, []) diff --git a/tests/calls/test_call_state_machine.py b/tests/calls/test_call_state_machine.py new file mode 100644 index 0000000..78b5eb9 --- /dev/null +++ b/tests/calls/test_call_state_machine.py @@ -0,0 +1,178 @@ +from whatsapp_chatbot_python.calls.state_machine import CallStateMachine +import unittest + +from whatsapp_chatbot_python.calls.models import ( + CallEndReason, + CallEvent, + CallSession, + CallState, + TERMINAL_STATES, +) + + +class CallStateMachineTest(unittest.TestCase): + def setUp(self) -> None: + self.fsm = CallStateMachine() + self.session = CallSession("sender@c.us", "sender@c.us", "ru") + + def apply(self, event: CallEvent) -> CallState: + self.fsm.apply(self.session, event) + return self.session.state + + def test_successful_call_requires_remote_and_bridge(self) -> None: + self.assertEqual(self.apply(CallEvent.ENQUEUED), CallState.QUEUED) + self.assertEqual(self.apply(CallEvent.DEQUEUED), CallState.DIALING) + self.assertEqual(self.apply(CallEvent.DIAL_ACCEPTED), CallState.RINGING) + result = self.fsm.apply(self.session, CallEvent.BRIDGE_READY) + self.assertTrue(result.applied) + self.assertFalse(result.changed) + self.assertEqual(self.session.state, CallState.RINGING) + self.assertTrue(self.session.bridge_ready) + self.assertEqual(self.apply(CallEvent.REMOTE_ACCEPTED), CallState.IN_CALL) + self.assertEqual(self.apply(CallEvent.REMOTE_IDLE), CallState.REMOTE_ENDED) + self.assertEqual(self.session.end_reason, CallEndReason.REMOTE_HANGUP) + + def test_remote_answer_before_bridge_uses_connecting_state(self) -> None: + for event in ( + CallEvent.ENQUEUED, + CallEvent.DEQUEUED, + CallEvent.DIAL_ACCEPTED, + CallEvent.REMOTE_ACCEPTED, + ): + self.apply(event) + self.assertEqual(self.session.state, CallState.CONNECTING) + self.assertEqual(self.apply(CallEvent.BRIDGE_READY), CallState.IN_CALL) + + def test_ring_timeout_has_separate_terminal_state(self) -> None: + for event in ( + CallEvent.ENQUEUED, + CallEvent.DEQUEUED, + CallEvent.DIAL_ACCEPTED, + CallEvent.RING_TIMER_EXPIRED, + ): + self.apply(event) + self.assertEqual(self.session.state, CallState.ENDING) + self.assertEqual(self.apply(CallEvent.HANGUP_CONFIRMED), CallState.RING_TIMEOUT) + + def test_talk_timeout_has_separate_terminal_state(self) -> None: + for event in ( + CallEvent.ENQUEUED, + CallEvent.DEQUEUED, + CallEvent.DIAL_ACCEPTED, + CallEvent.REMOTE_ACCEPTED, + CallEvent.BRIDGE_READY, + CallEvent.TALK_TIMER_EXPIRED, + ): + self.apply(event) + self.assertEqual(self.session.state, CallState.ENDING) + self.assertEqual(self.apply(CallEvent.HANGUP_CONFIRMED), CallState.TALK_TIMEOUT) + + def test_remote_hangup_after_answer(self) -> None: + for event in ( + CallEvent.ENQUEUED, + CallEvent.DEQUEUED, + CallEvent.DIAL_ACCEPTED, + CallEvent.REMOTE_ACCEPTED, + CallEvent.BRIDGE_READY, + CallEvent.REMOTE_IDLE, + ): + self.apply(event) + self.assertEqual(self.session.state, CallState.REMOTE_ENDED) + + def test_remote_idle_is_rejected_before_answer(self) -> None: + for event in ( + CallEvent.ENQUEUED, + CallEvent.DEQUEUED, + CallEvent.DIAL_ACCEPTED, + CallEvent.REMOTE_IDLE, + ): + self.apply(event) + self.assertEqual(self.session.state, CallState.REJECTED) + + def test_remote_idle_is_rejected_while_dialing(self) -> None: + self.apply(CallEvent.ENQUEUED) + self.apply(CallEvent.DEQUEUED) + + self.assertEqual(self.apply(CallEvent.REMOTE_IDLE), CallState.REJECTED) + self.assertEqual(self.session.end_reason, CallEndReason.REMOTE_REJECTED) + + def test_remote_hangup_while_connecting(self) -> None: + for event in ( + CallEvent.ENQUEUED, + CallEvent.DEQUEUED, + CallEvent.DIAL_ACCEPTED, + CallEvent.REMOTE_ACCEPTED, + ): + self.apply(event) + + self.assertEqual(self.apply(CallEvent.REMOTE_IDLE), CallState.REMOTE_ENDED) + self.assertEqual(self.session.end_reason, CallEndReason.REMOTE_HANGUP) + + def test_remote_idle_confirms_pending_timeout(self) -> None: + for event in ( + CallEvent.ENQUEUED, + CallEvent.DEQUEUED, + CallEvent.DIAL_ACCEPTED, + CallEvent.RING_TIMER_EXPIRED, + ): + self.apply(event) + + self.assertEqual(self.apply(CallEvent.REMOTE_IDLE), CallState.RING_TIMEOUT) + + def test_late_event_cannot_change_terminal_state(self) -> None: + self.test_remote_idle_is_rejected_before_answer() + result = self.fsm.apply(self.session, CallEvent.REMOTE_ACCEPTED) + self.assertFalse(result.applied) + self.assertFalse(result.changed) + self.assertEqual(self.session.state, CallState.REJECTED) + + def test_unsupported_event_is_ignored(self) -> None: + self.apply(CallEvent.ENQUEUED) + result = self.fsm.apply(self.session, CallEvent.BRIDGE_READY) + self.assertFalse(result.applied) + self.assertFalse(result.changed) + self.assertEqual(self.session.state, CallState.QUEUED) + + def test_internal_error_is_defined_for_every_nonterminal_state(self) -> None: + for state in CallState: + if state in TERMINAL_STATES: + continue + with self.subTest(state=state): + session = CallSession("sender@c.us", "sender@c.us", "ru", state=state) + result = self.fsm.apply( + session, + CallEvent.INTERNAL_ERROR, + error_code="test", + ) + self.assertTrue(result.applied) + self.assertEqual(session.state, CallState.FAILED) + self.assertEqual(session.end_reason, CallEndReason.ERROR) + + def test_shutdown_has_explicit_end_reason(self) -> None: + self.apply(CallEvent.ENQUEUED) + result = self.fsm.apply( + self.session, + CallEvent.SHUTDOWN_REQUESTED, + error_code="shutdown", + ) + self.assertTrue(result.applied) + self.assertEqual(self.session.state, CallState.FAILED) + self.assertEqual(self.session.end_reason, CallEndReason.SHUTDOWN) + + def test_all_terminal_states_ignore_late_events(self) -> None: + for state in TERMINAL_STATES: + for event in CallEvent: + with self.subTest(state=state, event=event): + session = CallSession( + "sender@c.us", + "sender@c.us", + "ru", + state=state, + ) + result = self.fsm.apply(session, event) + self.assertFalse(result.applied) + self.assertEqual(session.state, state) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/calls/test_playback_state_machine.py b/tests/calls/test_playback_state_machine.py new file mode 100644 index 0000000..0aa1ab0 --- /dev/null +++ b/tests/calls/test_playback_state_machine.py @@ -0,0 +1,43 @@ +import unittest + +from whatsapp_chatbot_python.calls.playback_state_machine import ( + PlaybackEvent, + PlaybackState, + PlaybackStateMachine, +) + + +class PlaybackStateMachineTest(unittest.TestCase): + def test_allowed_transitions(self): + fsm = PlaybackStateMachine() + + transitions = [ + (PlaybackState.PLAYING, PlaybackEvent.SOFT_LIMIT, PlaybackState.BACKLOG), + (PlaybackState.BACKLOG, PlaybackEvent.RECOVERED, PlaybackState.PLAYING), + (PlaybackState.PLAYING, PlaybackEvent.HARD_LIMIT, PlaybackState.DRAINING), + (PlaybackState.BACKLOG, PlaybackEvent.HARD_LIMIT, PlaybackState.DRAINING), + (PlaybackState.DRAINING, PlaybackEvent.DRAINED, PlaybackState.PLAYING), + (PlaybackState.SUSPENDED, PlaybackEvent.RESUMED, PlaybackState.PLAYING), + ] + + for state in (PlaybackState.PLAYING, PlaybackState.BACKLOG, PlaybackState.DRAINING): + transitions.extend([ + (state, PlaybackEvent.STALLED, PlaybackState.SUSPENDED), + (state, PlaybackEvent.INTERRUPTED, PlaybackState.PLAYING), + ]) + + for state in PlaybackState: + transitions.append((state, PlaybackEvent.TRACK_REPLACED, PlaybackState.PLAYING)) + + for previous, event, expected in transitions: + with self.subTest(previous=previous, event=event): + self.assertEqual(fsm.apply(previous, event), expected) + + def test_duplicate_and_out_of_order_events_are_ignored(self): + fsm = PlaybackStateMachine() + + self.assertEqual(fsm.apply(PlaybackState.DRAINING, PlaybackEvent.SOFT_LIMIT), + PlaybackState.DRAINING) + + self.assertEqual(fsm.apply(PlaybackState.SUSPENDED, PlaybackEvent.DRAINED), + PlaybackState.SUSPENDED) diff --git a/tests/calls/test_realtime_voice.py b/tests/calls/test_realtime_voice.py new file mode 100644 index 0000000..c9f1401 --- /dev/null +++ b/tests/calls/test_realtime_voice.py @@ -0,0 +1,455 @@ +from fractions import Fraction +from types import SimpleNamespace +from unittest.mock import ANY, patch +from whatsapp_chatbot_python.calls.playback_state_machine import PlaybackState +import asyncio +import base64 +import unittest +import av + +from whatsapp_chatbot_python.calls.realtime_voice import ( + BotOutputTrack, + CallerAudioSink, + VoiceBotSession, + FRAME_BYTES, + FRAME_DURATION_MS, + HARD_PLAYBACK_BUFFER_MS, + SOFT_PLAYBACK_BUFFER_MS, + RECOVER_PLAYBACK_BUFFER_MS, +) + + +class FakeConnection: + def __init__(self): + self.events = asyncio.Queue() + self.actions = [] + self.session = SimpleNamespace(update=self.update) + self.response = SimpleNamespace(create=self.create, cancel=self.cancel) + self.input_audio_buffer = SimpleNamespace(append=self.append) + self.conversation = SimpleNamespace(item=SimpleNamespace(truncate=self.truncate)) + + async def update(self, **kwargs): + self.actions.append(("update", kwargs)) + await self.events.put(SimpleNamespace(type="session.updated")) + + async def create(self, **kwargs): + self.actions.append(("greet", kwargs)) + + async def cancel(self, **kwargs): + self.actions.append(("cancel", kwargs)) + + async def append(self, **kwargs): + self.actions.append(("append", kwargs)) + + async def truncate(self, **kwargs): + self.actions.append(("truncate", kwargs)) + + async def recv(self): + return await self.events.get() + + async def __aiter__(self): + while True: + yield await self.recv() + + +class FakeManager: + def __init__(self, conn): + self.conn = conn + self.closed = False + + async def __aenter__(self): + return self.conn + + async def __aexit__(self, *args): + self.closed = True + + +class FakeOpenAI: + def __init__(self, api_key): + self.api_key = api_key + self.connection = FakeConnection() + self.manager = FakeManager(self.connection) + self.realtime = SimpleNamespace(connect=lambda **kwargs: self.manager) + self.closed = False + + async def close(self): + self.closed = True + + +class OneFrameTrack: + kind = "audio" + + def __init__(self): + self.sent = False + + async def recv(self): + if self.sent: + await asyncio.Future() + + self.sent = True + frame = av.AudioFrame(format="s16", layout="mono", samples=480) + frame.planes[0].update(bytes(FRAME_BYTES)) + frame.sample_rate = 24_000 + frame.pts = 0 + frame.time_base = Fraction(1, 24_000) + + return frame + + +class VoiceSessionTest(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.client = None + + def make_client(**kwargs): + self.client = FakeOpenAI(**kwargs) + return self.client + + patcher = patch("whatsapp_chatbot_python.calls.realtime_voice.AsyncOpenAI", make_client) + + patcher.start() + self.addCleanup(patcher.stop) + self.failures = [] + + self.voice = VoiceBotSession( + api_key="test", model="gpt-realtime-2.1", voice="marin", + language="ru", on_error=self.failures.append, + ) + + async def asyncTearDown(self): + await self.voice.close() + + async def test_session_configuration_and_single_greeting(self): + await self.voice.start() + await self.voice.greet_once() + await self.voice.greet_once() + + actions = self.client.connection.actions + + self.assertEqual([a for a, _ in actions], ["update", "greet"]) + self.assertEqual(actions[0][1]["session"]["audio"]["input"]["format"]["rate"], 24_000) + + async def test_supported_languages_configure_voice_and_greeting(self): + for code, language in { + "ru": "Russian", "en": "English", "he": "Hebrew", + "es": "Spanish", "kz": "Kazakh", + }.items(): + with self.subTest(code=code): + voice = VoiceBotSession( + api_key="test", model="gpt-realtime-2.1", voice="marin", + language=code, on_error=self.failures.append, + ) + + try: + await voice.start() + await voice.greet_once() + + actions = self.client.connection.actions + + self.assertIn(language, actions[0][1]["session"]["instructions"]) + self.assertIn(language, actions[1][1]["response"]["instructions"]) + finally: + await voice.close() + + async def test_audio_frames_are_paced_and_timestamped(self): + await self.voice.start() + + track = await self.voice.new_output_track() + + self.voice._queue_audio("item-1", bytes([1, 2]) * 480) + + first = await track.recv() + second = await track.recv() + + self.assertEqual(first.pts, 0) + self.assertEqual(second.pts, 480) + self.assertEqual(bytes(first.planes[0])[:FRAME_BYTES], bytes([1, 2]) * 480) + self.assertEqual(bytes(second.planes[0])[:FRAME_BYTES], bytes(FRAME_BYTES)) + self.assertEqual(track._voice._played["item-1"], 480) + + async def test_barge_in_clears_audio_and_truncates_played_part(self): + await self.voice.start() + + track = await self.voice.new_output_track() + + self.voice._active_responses.add("response-1") + self.voice._item_response["item-1"] = "response-1" + self.voice._queue_audio("item-1", bytes(FRAME_BYTES * 3)) + await track.recv() + await self.voice._interrupt(cancel=True) + self.assertEqual(len(self.voice._frames), 0) + + self.assertIn(("cancel", {"response_id": "response-1", "event_id": ANY}), + self.client.connection.actions) + self.assertIn(("truncate", { + "item_id": "item-1", "content_index": 0, "audio_end_ms": 20, + }), self.client.connection.actions) + + self.voice._queue_audio("item-1", bytes(FRAME_BYTES)) + self.assertEqual(len(self.voice._frames), 0) + + async def test_backlog_recovers_without_cancelling_generation(self): + await self.voice.start() + track = await self.voice.new_output_track() + self.voice._queue_audio("item-1", bytes( + FRAME_BYTES * (SOFT_PLAYBACK_BUFFER_MS // FRAME_DURATION_MS) + )) + self.assertEqual(self.voice.playback_state, PlaybackState.BACKLOG) + + while len(self.voice._frames) * FRAME_DURATION_MS > RECOVER_PLAYBACK_BUFFER_MS: + self.voice.take_frame(track._generation) + + self.assertEqual(self.voice.playback_state, PlaybackState.PLAYING) + self.assertFalse(any(action == "cancel" for action, _ in self.client.connection.actions)) + + async def test_full_buffer_drains_accepted_audio_before_truncating(self): + await self.voice.start() + + old = await self.voice.new_output_track() + await self.client.connection.events.put(SimpleNamespace( + type="response.created", response=SimpleNamespace(id="response-1") + )) + await self.client.connection.events.put(SimpleNamespace( + type="response.output_audio.delta", response_id="response-1", + item_id="item-1", content_index=0, + delta=base64.b64encode( + bytes([1, 2]) * (FRAME_BYTES // 2) + * (HARD_PLAYBACK_BUFFER_MS // FRAME_DURATION_MS + 1) + ).decode(), + )) + + for _ in range(20): + if any(action == "cancel" for action, _ in self.client.connection.actions): + break + await asyncio.sleep(0.01) + + actions = self.client.connection.actions + self.assertEqual(actions[-1][0], "cancel") + self.assertEqual(actions[-1][1]["response_id"], "response-1") + self.assertEqual(self.voice.playback_state, PlaybackState.DRAINING) + self.assertEqual(len(self.voice._frames) * FRAME_DURATION_MS, HARD_PLAYBACK_BUFFER_MS) + self.assertEqual(self.failures, []) + + await self.client.connection.events.put(SimpleNamespace( + type="response.output_audio.delta", response_id="response-1", + item_id="item-1", content_index=0, + delta=base64.b64encode(bytes(FRAME_BYTES)).decode(), + )) + await asyncio.sleep(0) + self.assertEqual(len(self.voice._frames) * FRAME_DURATION_MS, HARD_PLAYBACK_BUFFER_MS) + + for _ in range(HARD_PLAYBACK_BUFFER_MS // FRAME_DURATION_MS): + self.assertEqual(self.voice.take_frame(old._generation), bytes([1, 2]) * 480) + + for _ in range(20): + if any(action == "truncate" for action, _ in self.client.connection.actions): + break + await asyncio.sleep(0.01) + + self.assertEqual(self.voice.playback_state, PlaybackState.PLAYING) + self.assertEqual(self.client.connection.actions[-1], ("truncate", { + "item_id": "item-1", "content_index": 0, + "audio_end_ms": HARD_PLAYBACK_BUFFER_MS, + })) + + new = await self.voice.new_output_track() + + self.assertIsInstance(new, BotOutputTrack) + self.assertEqual(self.voice.take_frame(old._generation), bytes(FRAME_BYTES)) + self.assertEqual(len(self.voice._frames), 0) + + await self.client.connection.events.put(SimpleNamespace( + type="response.created", response=SimpleNamespace(id="response-2") + )) + await self.client.connection.events.put(SimpleNamespace( + type="response.output_audio.delta", response_id="response-2", + item_id="item-2", content_index=0, + delta=base64.b64encode(bytes([1, 2]) * 480).decode(), + )) + for _ in range(20): + if self.voice._frames: + break + await asyncio.sleep(0.01) + self.assertEqual(bytes((await new.recv()).planes[0])[:FRAME_BYTES], bytes([1, 2]) * 480) + self.assertEqual(self.failures, []) + + async def test_barge_in_during_drain_discards_pending_audio(self): + await self.voice.start() + track = await self.voice.new_output_track() + await self.client.connection.events.put(SimpleNamespace( + type="response.created", response=SimpleNamespace(id="response-1") + )) + await self.client.connection.events.put(SimpleNamespace( + type="response.output_audio.delta", response_id="response-1", + item_id="item-1", content_index=0, + delta=base64.b64encode(bytes( + FRAME_BYTES * (HARD_PLAYBACK_BUFFER_MS // FRAME_DURATION_MS + 1) + )).decode(), + )) + for _ in range(20): + if self.voice.playback_state == PlaybackState.DRAINING: + break + await asyncio.sleep(0.01) + + self.voice.take_frame(track._generation) + await self.client.connection.events.put(SimpleNamespace( + type="input_audio_buffer.speech_started" + )) + for _ in range(20): + if any(action == "truncate" for action, _ in self.client.connection.actions): + break + await asyncio.sleep(0.01) + + self.assertEqual(self.voice.playback_state, PlaybackState.PLAYING) + self.assertEqual(len(self.voice._frames), 0) + self.assertEqual(self.client.connection.actions[-1][1]["audio_end_ms"], FRAME_DURATION_MS) + self.assertEqual(self.failures, []) + + async def test_track_replacement_truncates_old_playback(self): + await self.voice.start() + old = await self.voice.new_output_track() + await self.client.connection.events.put(SimpleNamespace( + type="response.created", response=SimpleNamespace(id="response-1") + )) + await self.client.connection.events.put(SimpleNamespace( + type="response.output_audio.delta", response_id="response-1", + item_id="item-1", content_index=0, + delta=base64.b64encode(bytes(FRAME_BYTES * 3)).decode(), + )) + for _ in range(20): + if self.voice._frames: + break + await asyncio.sleep(0.01) + + self.voice.take_frame(old._generation) + new = await self.voice.new_output_track() + + self.assertEqual(self.voice.take_frame(old._generation), bytes(FRAME_BYTES)) + self.assertEqual(len(self.voice._frames), 0) + self.assertEqual(self.voice.playback_state, PlaybackState.PLAYING) + self.assertIn(("truncate", { + "item_id": "item-1", "content_index": 0, + "audio_end_ms": FRAME_DURATION_MS, + }), self.client.connection.actions) + self.assertIsInstance(new, BotOutputTrack) + self.assertEqual(self.failures, []) + + async def test_stalled_consumer_is_recovered_without_failing_call(self): + await self.voice.start() + track = await self.voice.new_output_track() + await self.client.connection.events.put(SimpleNamespace( + type="response.created", response=SimpleNamespace(id="response-1") + )) + await self.client.connection.events.put(SimpleNamespace( + type="response.output_audio.delta", response_id="response-1", + item_id="item-1", content_index=0, + delta=base64.b64encode(bytes(FRAME_BYTES * 200)).decode(), + )) + + # Advance only the playback clock; keep the production watchdog interval intact. + for _ in range(20): + if self.voice._buffered_since is not None: + break + await asyncio.sleep(0.01) + self.assertIsNotNone(self.voice._buffered_since) + self.voice._buffered_since -= SOFT_PLAYBACK_BUFFER_MS / 1000 + 1 + await asyncio.sleep(0.55) + + self.assertEqual([action for action, _ in self.client.connection.actions[-2:]], + ["cancel", "truncate"]) + self.assertEqual(len(self.voice._frames), 0) + self.assertEqual(self.voice.playback_state, PlaybackState.SUSPENDED) + self.assertEqual(self.voice.take_frame(track._generation), bytes(FRAME_BYTES)) + self.assertEqual(self.voice.playback_state, PlaybackState.PLAYING) + self.assertEqual(self.failures, []) + + async def test_inbound_audio_is_sent_as_base64_pcm(self): + await self.voice.start() + await self.voice.new_output_track() + + sink = CallerAudioSink(self.voice, self.voice.generation) + + await sink.attach(OneFrameTrack()) + + for _ in range(20): + if any(action == "append" for action, _ in self.client.connection.actions): + break + + await asyncio.sleep(0.01) + + await sink.close() + + appended = [data for action, data in self.client.connection.actions if action == "append"] + + self.assertTrue(appended) + self.assertEqual(len(base64.b64decode(appended[0]["audio"])), FRAME_BYTES) + self.assertEqual(self.failures, []) + + async def test_recording_taps_tracks_and_ignores_old_bridge(self): + class Recorder: + def __init__(self): + self.generation = 0 + self.caller = [] + self.bot = [] + + def set_generation(self, generation): + self.generation = generation + + def add_caller(self, pcm, generation): + if generation == self.generation: + self.caller.append(pcm) + + def add_bot(self, pcm, generation): + if generation == self.generation: + self.bot.append(pcm) + + recorder = Recorder() + + voice = VoiceBotSession( + api_key="test", + model="gpt-realtime-2.1", + voice="marin", + language="ru", + on_error=self.failures.append, + recorder=recorder, + ) + + try: + await voice.start() + + old_track = await voice.new_output_track() + sink = voice.new_input_sink() + + await sink.attach(OneFrameTrack()) + + for _ in range(20): + if recorder.caller: + break + + await asyncio.sleep(0.01) + + await sink.close() + voice._queue_audio("item-1", bytes([1, 2]) * 480) + await old_track.recv() + self.assertEqual(recorder.caller, [bytes(FRAME_BYTES)]) + self.assertEqual(recorder.bot, [bytes([1, 2]) * 480]) + await voice.new_output_track() + await old_track.recv() + self.assertEqual(len(recorder.bot), 1) + finally: + await voice.close() + + async def test_failed_realtime_response_reports_call_error(self): + await self.voice.start() + + await self.client.connection.events.put(SimpleNamespace( + type="response.done", response=SimpleNamespace(status="failed") + )) + + for _ in range(20): + if self.failures: + break + + await asyncio.sleep(0.01) + + self.assertEqual(len(self.failures), 1) + self.assertIsInstance(self.failures[0], RuntimeError) diff --git a/tests/test_calls_imports.py b/tests/test_calls_imports.py new file mode 100644 index 0000000..7c0a413 --- /dev/null +++ b/tests/test_calls_imports.py @@ -0,0 +1,54 @@ +"""The optional call implementation must not affect ordinary chatbot imports.""" + +import subprocess +import sys +import unittest + + +class CallsImportTest(unittest.TestCase): + def run_script(self, script): + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_base_import_does_not_require_audio_or_openai(self): + self.run_script(''' +import importlib.abc +import sys + +class BlockOptionalImports(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split(".")[0] in {"openai", "aiortc", "av"}: + raise ImportError("optional dependency blocked: " + fullname) + +sys.meta_path.insert(0, BlockOptionalImports()) +from whatsapp_chatbot_python import GreenAPIBot, Notification +assert "whatsapp_chatbot_python.calls" not in sys.modules +''') + + @unittest.skipIf(sys.version_info < (3, 11), "call contracts require Python 3.11+") + def test_contracts_do_not_load_call_implementation(self): + self.run_script(''' +import sys +from whatsapp_chatbot_python.calls import CallSession, CallStateMachine +assert CallSession("sender", "chat", "en").state == "triggered" +assert "whatsapp_chatbot_python.calls.service" not in sys.modules +assert "whatsapp_chatbot_python.calls.realtime_voice" not in sys.modules +assert "openai" not in sys.modules +assert "aiortc" not in sys.modules +assert "av" not in sys.modules +''') + + def test_unsupported_python_has_explicit_error(self): + self.run_script(''' +import sys +import whatsapp_chatbot_python +sys.version_info = (3, 10, 0) +try: + import whatsapp_chatbot_python.calls +except ImportError as error: + assert "Python 3.11" in str(error) +else: + raise AssertionError("calls accepted an unsupported Python version") +''') diff --git a/whatsapp_chatbot_python/calls/__init__.py b/whatsapp_chatbot_python/calls/__init__.py new file mode 100644 index 0000000..198bfe1 --- /dev/null +++ b/whatsapp_chatbot_python/calls/__init__.py @@ -0,0 +1,34 @@ +"""Single-call contracts and state machine (Python 3.11+). + +Import the executor from ``.service`` after installing the ``voip`` extra. +The base chatbot package does not import this optional package. +""" + +# Check the runtime before importing modules that use StrEnum. +# ruff: noqa: E402 +import sys + +if sys.version_info < (3, 11): + raise ImportError("whatsapp_chatbot_python.calls requires Python 3.11 or newer") + +from .contracts import CallExecutor, TransitionCallback +from .models import ( + ACTIVE_STATES, + TERMINAL_STATES, + WAITING_STATES, + CallEndReason, + CallEvent, + CallExecutionResult, + CallSession, + CallState, + StateTransition, + TransitionOutcome, +) +from .state_machine import CallStateMachine + +__all__ = [ + "CallExecutor", "TransitionCallback", "CallEndReason", "CallEvent", + "CallExecutionResult", "CallSession", "CallState", "StateTransition", + "TransitionOutcome", "CallStateMachine", "ACTIVE_STATES", + "TERMINAL_STATES", "WAITING_STATES", +] diff --git a/whatsapp_chatbot_python/calls/contracts.py b/whatsapp_chatbot_python/calls/contracts.py new file mode 100644 index 0000000..730f992 --- /dev/null +++ b/whatsapp_chatbot_python/calls/contracts.py @@ -0,0 +1,28 @@ +"""Contracts shared by the executor and an application's call coordinator.""" + +from typing import Protocol + +from .models import CallEvent, CallExecutionResult, CallSession, StateTransition + + +class TransitionCallback(Protocol): + """Synchronously apply the event to the supplied session, under its owner's lock.""" + + def __call__( + self, + session: CallSession, + event: CallEvent, + *, + remote_reason: str | None = None, + error_code: str | None = None, + ) -> StateTransition: ... + + +class CallExecutor(Protocol): + """Execute calls serially; request_stop permanently stops this executor.""" + + async def execute( + self, session: CallSession, transition: TransitionCallback + ) -> CallExecutionResult: ... + + def request_stop(self) -> None: ... diff --git a/whatsapp_chatbot_python/calls/models.py b/whatsapp_chatbot_python/calls/models.py new file mode 100644 index 0000000..dde8092 --- /dev/null +++ b/whatsapp_chatbot_python/calls/models.py @@ -0,0 +1,121 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import StrEnum +from pathlib import Path +from uuid import uuid4 + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class CallState(StrEnum): + TRIGGERED = "triggered" + QUEUED = "queued" + CANCELLED = "cancelled" + DIALING = "dialing" + RINGING = "ringing" + CONNECTING = "connecting" + IN_CALL = "in_call" + ENDING = "ending" + REJECTED = "rejected" + RING_TIMEOUT = "ring_timeout" + REMOTE_ENDED = "remote_ended" + TALK_TIMEOUT = "talk_timeout" + FAILED = "failed" + + +class CallEvent(StrEnum): + ENQUEUED = "enqueued" + DEQUEUED = "dequeued" + CANCEL_REQUESTED = "cancel_requested" + DIAL_ACCEPTED = "dial_accepted" + REMOTE_ACCEPTED = "remote_accepted" + BRIDGE_READY = "bridge_ready" + REMOTE_IDLE = "remote_idle" + RING_TIMER_EXPIRED = "ring_timer_expired" + TALK_TIMER_EXPIRED = "talk_timer_expired" + HANGUP_CONFIRMED = "hangup_confirmed" + SHUTDOWN_REQUESTED = "shutdown_requested" + INTERNAL_ERROR = "internal_error" + + +class CallEndReason(StrEnum): + REMOTE_HANGUP = "remote_hangup" + REMOTE_REJECTED = "remote_rejected" + RING_TIMEOUT = "ring_timeout" + TALK_TIMEOUT = "talk_timeout" + ERROR = "error" + SHUTDOWN = "shutdown" + USER_CANCELLED = "user_cancelled" + + +class TransitionOutcome(StrEnum): + APPLIED = "applied" + IGNORED = "ignored" + + +TERMINAL_STATES = frozenset( + { + CallState.REJECTED, + CallState.RING_TIMEOUT, + CallState.REMOTE_ENDED, + CallState.TALK_TIMEOUT, + CallState.FAILED, + CallState.CANCELLED, + } +) + +WAITING_STATES = frozenset( + { + CallState.TRIGGERED, + CallState.QUEUED, + CallState.DIALING, + CallState.RINGING, + CallState.CONNECTING, + } +) + +ACTIVE_STATES = frozenset({CallState.IN_CALL, CallState.ENDING}) + + +@dataclass(slots=True) +class CallSession: + sender_id: str + chat_id: str + language: str + session_id: str = field(default_factory=lambda: str(uuid4())) + state: CallState = CallState.TRIGGERED + call_id: str | None = None + remote_reason: str | None = None + end_reason: CallEndReason | None = None + error_code: str | None = None + triggered_at: datetime = field(default_factory=utc_now) + queued_at: datetime | None = None + dialing_at: datetime | None = None + answered_at: datetime | None = None + ended_at: datetime | None = None + bridge_ready: bool = False + remote_accepted: bool = False + + +@dataclass(frozen=True, slots=True) +class StateTransition: + previous: CallState + current: CallState + event: CallEvent + outcome: TransitionOutcome + + @property + def applied(self) -> bool: + return self.outcome == TransitionOutcome.APPLIED + + @property + def changed(self) -> bool: + return self.current != self.previous + + +@dataclass(frozen=True, slots=True) +class CallExecutionResult: + recording_path: Path | None = None diff --git a/whatsapp_chatbot_python/calls/playback_state_machine.py b/whatsapp_chatbot_python/calls/playback_state_machine.py new file mode 100644 index 0000000..5091420 --- /dev/null +++ b/whatsapp_chatbot_python/calls/playback_state_machine.py @@ -0,0 +1,46 @@ +"""Allowed transitions for one call's outgoing audio playback.""" + +from enum import Enum + + +class PlaybackState(str, Enum): + PLAYING = "playing" + BACKLOG = "backlog" + DRAINING = "draining" + SUSPENDED = "suspended" + + +class PlaybackEvent(str, Enum): + SOFT_LIMIT = "soft_limit" + RECOVERED = "recovered" + HARD_LIMIT = "hard_limit" + DRAINED = "drained" + STALLED = "stalled" + RESUMED = "resumed" + INTERRUPTED = "interrupted" + TRACK_REPLACED = "track_replaced" + + +_TRANSITION_RULES: dict[tuple[PlaybackState, PlaybackEvent], PlaybackState] = { + (PlaybackState.PLAYING, PlaybackEvent.SOFT_LIMIT): PlaybackState.BACKLOG, + (PlaybackState.BACKLOG, PlaybackEvent.RECOVERED): PlaybackState.PLAYING, + (PlaybackState.PLAYING, PlaybackEvent.HARD_LIMIT): PlaybackState.DRAINING, + (PlaybackState.BACKLOG, PlaybackEvent.HARD_LIMIT): PlaybackState.DRAINING, + (PlaybackState.DRAINING, PlaybackEvent.DRAINED): PlaybackState.PLAYING, + (PlaybackState.SUSPENDED, PlaybackEvent.RESUMED): PlaybackState.PLAYING, +} + +for state in (PlaybackState.PLAYING, PlaybackState.BACKLOG, PlaybackState.DRAINING): + _TRANSITION_RULES[(state, PlaybackEvent.STALLED)] = PlaybackState.SUSPENDED + _TRANSITION_RULES[(state, PlaybackEvent.INTERRUPTED)] = PlaybackState.PLAYING + +for state in PlaybackState: + _TRANSITION_RULES[(state, PlaybackEvent.TRACK_REPLACED)] = PlaybackState.PLAYING + + +class PlaybackStateMachine: + """Ignore duplicate or late events, as the call state machine does.""" + + @staticmethod + def apply(state: PlaybackState, event: PlaybackEvent) -> PlaybackState: + return _TRANSITION_RULES.get((state, event), state) diff --git a/whatsapp_chatbot_python/calls/realtime_voice.py b/whatsapp_chatbot_python/calls/realtime_voice.py new file mode 100644 index 0000000..34e45a3 --- /dev/null +++ b/whatsapp_chatbot_python/calls/realtime_voice.py @@ -0,0 +1,520 @@ +"""One OpenAI Realtime conversation per Green-API call.""" + +from __future__ import annotations +from collections import deque +from fractions import Fraction +from time import monotonic +from typing import Callable +from uuid import uuid4 +from aiortc import AudioStreamTrack, MediaStreamTrack +from aiortc.mediastreams import MediaStreamError +from openai import AsyncOpenAI +from .playback_state_machine import PlaybackEvent, PlaybackState, PlaybackStateMachine +from .recording import CallRecorder +import asyncio +import base64 +import logging +import av + + +# Constants + +AUDIO_SAMPLE_RATE_HZ = 24 * 1000 # 24000 hz + +FRAME_DURATION_MS = 20 # 0.02 seconds + +SOFT_PLAYBACK_BUFFER_MS = 6 * 1000 # 6 seconds + +HARD_PLAYBACK_BUFFER_MS = 24 * 1000 # 24 seconds + +FRAME_SAMPLES = AUDIO_SAMPLE_RATE_HZ * FRAME_DURATION_MS // 1000 # 480 samples + +RECOVER_PLAYBACK_BUFFER_MS = SOFT_PLAYBACK_BUFFER_MS // 2 # 3 second + +FRAME_BYTES = 2 * FRAME_SAMPLES # 960 bytes (Mono PCM16) + +LANGUAGE_NAMES = { + "ru": "Russian", + "en": "English", + "he": "Hebrew", + "es": "Spanish", + "kz": "Kazakh", +} + + +# Variables + +logger = logging.getLogger(__name__) + + +class BotOutputTrack(AudioStreamTrack): + def __init__(self, voice: VoiceBotSession, generation: int): + super().__init__() + self._voice = voice + self._generation = generation + self._pts = 0 + self._next_frame_at: float | None = None + + async def recv(self) -> av.AudioFrame: + loop = asyncio.get_running_loop() + + if self._next_frame_at is None: + self._next_frame_at = loop.time() + + await asyncio.sleep(max(0, self._next_frame_at - loop.time())) + + self._next_frame_at = max(self._next_frame_at + FRAME_DURATION_MS / 1000, loop.time()) + + pcm = self._voice.take_frame(self._generation) + + if self._voice.recorder is not None: + self._voice.recorder.add_bot(pcm, self._generation) + + frame = av.AudioFrame(format="s16", layout="mono", samples=FRAME_SAMPLES) + + frame.planes[0].update(pcm) + + frame.sample_rate = AUDIO_SAMPLE_RATE_HZ + frame.time_base = Fraction(1, AUDIO_SAMPLE_RATE_HZ) + frame.pts = self._pts + self._pts += FRAME_SAMPLES + + return frame + + +class CallerAudioSink: + def __init__(self, voice: VoiceBotSession, generation: int): + self._voice = voice + self._generation = generation + self._task: asyncio.Task | None = None + + async def attach(self, track: MediaStreamTrack) -> None: + if track.kind != "audio": + raise ValueError("Expected an audio track") + + self._task = asyncio.create_task(self._consume(track)) + + async def _consume(self, track: MediaStreamTrack) -> None: + resampler = av.AudioResampler(format="s16", layout="mono", rate=AUDIO_SAMPLE_RATE_HZ) + + try: + while self._generation == self._voice.generation: + frame = await track.recv() + + for converted in resampler.resample(frame): + if self._generation != self._voice.generation: + return + + pcm = bytes(converted.planes[0])[:converted.samples * 2] + + if self._voice.recorder is not None: + self._voice.recorder.add_caller(pcm, self._generation) + + await self._voice.append_input(pcm) + except MediaStreamError: + pass + except asyncio.CancelledError: + raise + except Exception as error: + self._voice.fail(error) + + async def close(self) -> None: + if self._task is not None: + self._task.cancel() + await asyncio.gather(self._task, return_exceptions=True) + + self._task = None + + +class VoiceBotSession: + def __init__( + self, + *, + api_key: str, + model: str, + voice: str, + language: str, + on_error: Callable[[Exception], None], + recorder: CallRecorder | None = None, + ) -> None: + self._client = AsyncOpenAI(api_key=api_key) + self._model = model + self._voice = voice + self._language = language + self._on_error = on_error + self.recorder = recorder + self._manager = None + self._connection = None + self._reader: asyncio.Task | None = None + self._watchdog: asyncio.Task | None = None + self._drain_task: asyncio.Task | None = None + self._closed = False + self._greeted = False + self.generation = 0 + self._frames: deque[tuple[str | None, bytes]] = deque() + self._partial = bytearray() + self._last_item: str | None = None + self._played: dict[str, int] = {} + self._truncated: set[str] = set() + self._finished: set[str] = set() + self._item_response: dict[str, str] = {} + self._item_content_index: dict[str, int] = {} + self._active_responses: set[str] = set() + self._discarded_responses: set[str] = set() + self._cancel_events: set[str] = set() + self.playback_state = PlaybackState.PLAYING + self._playback_fsm = PlaybackStateMachine() + self._pending_truncation: set[str] = set() + self._buffered_since: float | None = None + self._last_pull_at: float | None = None + + async def start(self) -> None: + if not self._client.api_key: + raise ValueError("OpenAI API key is required for voice calls") + + self._manager = self._client.realtime.connect(model=self._model, max_retries=0) + self._connection = await self._manager.__aenter__() + + try: + await self._connection.session.update(session={ + "type": "realtime", + "model": self._model, + "output_modalities": ["audio"], + "instructions": ( + "You are a helpful voice assistant in a WhatsApp phone call. " + f"Speak in {LANGUAGE_NAMES.get(self._language, 'English')}. " + "Speak naturally and briefly. Do not use markdown." + ), + "audio": { + "input": { + "format": {"type": "audio/pcm", "rate": AUDIO_SAMPLE_RATE_HZ}, + "turn_detection": {"type": "semantic_vad"}, + }, + "output": { + "format": {"type": "audio/pcm", "rate": AUDIO_SAMPLE_RATE_HZ}, + "voice": self._voice, + }, + }, + }) + + while True: + event = await asyncio.wait_for(self._connection.recv(), timeout=15) + + if event.type == "session.updated": + break + + if event.type == "error": + raise RuntimeError("OpenAI Realtime rejected session configuration") + + self._reader = asyncio.create_task(self._read_events()) + self._watchdog = asyncio.create_task(self._watch_playback()) + except BaseException: + await self.close() + raise + + async def greet_once(self) -> None: + if self._greeted or self._closed: + return + + self._greeted = True + + await self._connection.response.create(response={ + "instructions": ( + f"Greet the caller briefly in {LANGUAGE_NAMES.get(self._language, 'English')}." + ), + }) + + async def new_output_track(self) -> BotOutputTrack: + if self.generation: + await self._interrupt(cancel=True, reason="track_replaced") + + self.generation += 1 + + if self.recorder is not None: + self.recorder.set_generation(self.generation) + + self._frames.clear() + self._partial.clear() + + self._last_item = None + self._buffered_since = None + self._last_pull_at = None + self._transition_playback(PlaybackEvent.TRACK_REPLACED) + + return BotOutputTrack(self, self.generation) + + def new_input_sink(self) -> CallerAudioSink: + return CallerAudioSink(self, self.generation) + + async def append_input(self, pcm: bytes) -> None: + if pcm and not self._closed: + await self._connection.input_audio_buffer.append( + audio=base64.b64encode(pcm).decode("ascii") + ) + + def _transition_playback(self, event: PlaybackEvent) -> None: + previous = self.playback_state + self.playback_state = self._playback_fsm.apply(previous, event) + + if self.playback_state != previous: + logger.info( + "OpenAI playback state: %s -> %s event=%s queued_ms=%d", + previous.value, self.playback_state.value, event.value, + len(self._frames) * FRAME_DURATION_MS, + ) + + def take_frame(self, generation: int) -> bytes: + if generation != self.generation: + return bytes(FRAME_BYTES) + + self._last_pull_at = monotonic() + + if self.playback_state == PlaybackState.SUSPENDED: + self._transition_playback(PlaybackEvent.RESUMED) + + if not self._frames: + return bytes(FRAME_BYTES) + + item, pcm = self._frames.popleft() + + if not self._frames and not self._partial: + self._buffered_since = None + + if self.playback_state == PlaybackState.DRAINING and self._drain_task is None: + self._drain_task = asyncio.create_task(self._finish_draining()) + + if ( + self.playback_state == PlaybackState.BACKLOG and + len(self._frames) * FRAME_DURATION_MS <= RECOVER_PLAYBACK_BUFFER_MS + ): + self._transition_playback(PlaybackEvent.RECOVERED) + + if item is not None: + self._played[item] = self._played.get(item, 0) + FRAME_SAMPLES + + return pcm + + def _queue_audio(self, item_id: str, pcm: bytes) -> bool: + if item_id in self._truncated: + return True + + if self._last_item != item_id: + self._partial.clear() + + self._last_item = item_id + + remaining_bytes = max( + 0, + (HARD_PLAYBACK_BUFFER_MS // FRAME_DURATION_MS - len(self._frames)) + * FRAME_BYTES - len(self._partial), + ) + + overflow = len(pcm) > remaining_bytes + pcm = pcm[:remaining_bytes] + + if not self._frames and not self._partial and pcm: + self._buffered_since = monotonic() + + self._partial.extend(pcm) + + while len(self._partial) >= FRAME_BYTES: + self._frames.append((item_id, bytes(self._partial[:FRAME_BYTES]))) + + del self._partial[:FRAME_BYTES] + + if len(self._frames) * FRAME_DURATION_MS >= SOFT_PLAYBACK_BUFFER_MS: + self._transition_playback(PlaybackEvent.SOFT_LIMIT) + + return not overflow + + def _finish_audio(self, item_id: str) -> bool: + if item_id in self._truncated: + return True + + self._finished.add(item_id) + + if self._partial and self._last_item == item_id: + if (len(self._frames) + 1) * FRAME_DURATION_MS > HARD_PLAYBACK_BUFFER_MS: + return False + + pcm = bytes(self._partial).ljust(FRAME_BYTES, b"\x00") + + self._frames.append((item_id, pcm)) + + self._partial.clear() + + return True + + async def _stop_generation(self, responses: set[str]) -> None: + new_responses = responses - self._discarded_responses + + self._discarded_responses.update(responses) + + for response_id in new_responses & self._active_responses: + event_id = f"playback-cancel-{uuid4().hex}" + self._cancel_events.add(event_id) + await self._connection.response.cancel(response_id=response_id, event_id=event_id) + + async def _truncate_items(self, items: set[str]) -> None: + pending = items - self._truncated + + self._truncated.update(pending) + + for item in pending: + await self._connection.conversation.item.truncate( + item_id=item, + content_index=self._item_content_index.get(item, 0), + audio_end_ms=self._played.get(item, 0) * 1000 // AUDIO_SAMPLE_RATE_HZ, + ) + + async def _finish_draining(self) -> None: + try: + items = set(self._pending_truncation) + + self._pending_truncation.clear() + await self._truncate_items(items) + self._transition_playback(PlaybackEvent.DRAINED) + except Exception as error: + self.fail(error) + finally: + self._drain_task = None + + async def _interrupt(self, *, cancel: bool = False, reason: str = "speech") -> None: + items = {item for item, _ in self._frames if item is not None} + items.update(self._pending_truncation) + + if self._last_item is not None and ( + self._partial or self._last_item not in self._finished + ): + items.add(self._last_item) + + responses = {self._item_response[item] for item in items if item in self._item_response} + + if cancel and (items or self._active_responses): + logger.warning( + "OpenAI playback interrupted: reason=%s queued_ms=%d pull_age_ms=%d", + reason, len(self._frames) * FRAME_DURATION_MS, + int((monotonic() - self._last_pull_at) * 1000) if self._last_pull_at else -1, + ) + + self._frames.clear() + self._partial.clear() + self._pending_truncation.clear() + self._last_item = None + self._buffered_since = None + + self._transition_playback( + PlaybackEvent.STALLED if reason == "stalled" else PlaybackEvent.INTERRUPTED + ) + + if cancel: + await self._stop_generation(responses | self._active_responses) + else: + self._discarded_responses.update(responses) + + await self._truncate_items(items) + + async def _hard_limit(self, response_id: str, item_id: str) -> None: + self._partial.clear() + self._pending_truncation.add(item_id) + self._transition_playback(PlaybackEvent.HARD_LIMIT) + await self._stop_generation({response_id}) + + async def _watch_playback(self) -> None: + try: + while not self._closed: + await asyncio.sleep(0.5) + + buffered_ms = len(self._frames) * FRAME_DURATION_MS + + if buffered_ms >= RECOVER_PLAYBACK_BUFFER_MS and self._buffered_since is not None: + last_progress = max(self._buffered_since, self._last_pull_at or 0) + + if (monotonic() - last_progress) * 1000 >= SOFT_PLAYBACK_BUFFER_MS: + await self._interrupt(cancel=True, reason="stalled") + except asyncio.CancelledError: + raise + except Exception as error: + self.fail(error) + + async def _read_events(self) -> None: + try: + async for event in self._connection: + if event.type == "response.created": + self._active_responses.add(event.response.id) + elif event.type == "response.output_audio.delta": + if event.response_id in self._discarded_responses: + self._item_response[event.item_id] = event.response_id + self._item_content_index[event.item_id] = event.content_index + + if event.item_id not in self._pending_truncation: + await self._truncate_items({event.item_id}) + + continue + + self._item_response[event.item_id] = event.response_id + self._item_content_index[event.item_id] = event.content_index + + if self.playback_state in {PlaybackState.DRAINING, PlaybackState.SUSPENDED}: + await self._stop_generation({event.response_id}) + await self._truncate_items({event.item_id}) + continue + + if not self._queue_audio(event.item_id, base64.b64decode(event.delta)): + await self._hard_limit(event.response_id, event.item_id) + elif event.type == "response.output_audio.done": + if self._item_response.get(event.item_id) in self._discarded_responses: + continue + + if not self._finish_audio(event.item_id): + await self._hard_limit(self._item_response[event.item_id], event.item_id) + elif event.type == "input_audio_buffer.speech_started": + await self._interrupt(cancel=True) + elif event.type == "response.done": + response_id = getattr(event.response, "id", None) + self._active_responses.discard(response_id) + + if event.response.status == "failed" and response_id not in self._discarded_responses: + raise RuntimeError("OpenAI Realtime response failed") + elif event.type == "error": + cancel_id = getattr(event.error, "event_id", None) + + if cancel_id in self._cancel_events: + self._cancel_events.discard(cancel_id) + continue + + raise RuntimeError("OpenAI Realtime reported an error") + + if not self._closed: + raise ConnectionError("OpenAI Realtime connection closed") + except asyncio.CancelledError: + raise + except Exception as error: + self.fail(error) + + def fail(self, error: Exception) -> None: + if not self._closed: + self._on_error(error) + + async def close(self) -> None: + if self._closed: + return + + self._closed = True + + if self._reader is not None: + self._reader.cancel() + await asyncio.gather(self._reader, return_exceptions=True) + + if self._watchdog is not None: + self._watchdog.cancel() + await asyncio.gather(self._watchdog, return_exceptions=True) + + if self._drain_task is not None: + self._drain_task.cancel() + await asyncio.gather(self._drain_task, return_exceptions=True) + + if self._manager is not None: + await self._manager.__aexit__(None, None, None) + + await self._client.close() diff --git a/whatsapp_chatbot_python/calls/recording.py b/whatsapp_chatbot_python/calls/recording.py new file mode 100644 index 0000000..f44fa12 --- /dev/null +++ b/whatsapp_chatbot_python/calls/recording.py @@ -0,0 +1,151 @@ +"""Best-effort recording of both sides of one call.""" + +from __future__ import annotations +from array import array +from pathlib import Path +from queue import Full, Queue +from tempfile import TemporaryFile, NamedTemporaryFile +from threading import Event, Thread +from time import monotonic +import logging +import av + + +# Constants + +RATE = 24 * 1000 + +MAX_PENDING_CHUNKS = 256 + +MIX_SAMPLES = 24 * 1000 + + +class CallRecorder: + def __init__(self, logger: logging.Logger) -> None: + self._logger = logger + self._queue: Queue[tuple[str, int, bytes] | None] = Queue(MAX_PENDING_CHUNKS) + self._started_at: float | None = None + self._generation = 0 + self._failed = Event() + self._thread: Thread | None = None + self._caller = TemporaryFile() + self._bot = TemporaryFile() + self._lengths = {"caller": 0, "bot": 0} + self._has_audio = False + + def start(self) -> None: + if self._started_at is None: + self._started_at = monotonic() + self._thread = Thread(target=self._write, name="voip-recording", daemon=True) + + self._thread.start() + + def set_generation(self, generation: int) -> None: + self._generation = generation + + def add_caller(self, pcm: bytes, generation: int) -> None: + self._add("caller", pcm, generation) + + def add_bot(self, pcm: bytes, generation: int) -> None: + self._add("bot", pcm, generation) + + def _add(self, side: str, pcm: bytes, generation: int) -> None: + if self._started_at is None or self._failed.is_set() or generation != self._generation or not pcm: + return + + if len(pcm) % 2: + self._failed.set() + return + + # Incoming frames arrive after capture; place their start at the beginning + # of the received interval. The outgoing frame starts when recv() yields it. + elapsed = monotonic() - self._started_at + + if side == "caller": + elapsed -= len(pcm) / (2 * RATE) + + offset = max(0, round(elapsed * RATE)) + + try: + self._queue.put_nowait((side, offset, pcm)) + except Full: + self._failed.set() + self._logger.warning("VoIP recording queue overflow") + + def _write(self) -> None: + streams = {"caller": self._caller, "bot": self._bot} + + try: + while (item := self._queue.get()) is not None: + side, offset, pcm = item + stream = streams[side] + stream.seek(offset * 2) + stream.write(pcm) + self._lengths[side] = max(self._lengths[side], offset + len(pcm) // 2) + self._has_audio = True + self._queue.task_done() + + self._queue.task_done() + except Exception: + self._failed.set() + self._logger.exception("VoIP recording write failed") + + def finish(self) -> Path | None: + """Finish queued writes and return an MP3 owned by the caller.""" + + if self._thread is not None: + while self._thread.is_alive(): + try: + self._queue.put(None, timeout=0.1) + break + except Full: + pass + + self._thread.join() + try: + if self._failed.is_set() or not self._has_audio: + return None + + with NamedTemporaryFile(prefix="voip-call-", suffix=".mp3", delete=False) as output: + path = Path(output.name) + + try: + self._encode(path) + return path + except Exception: + path.unlink(missing_ok=True) + self._logger.exception("VoIP recording encoding failed") + + return None + finally: + self._caller.close() + self._bot.close() + + def _encode(self, path: Path) -> None: + length = max(self._lengths.values()) + + self._caller.seek(0) + self._bot.seek(0) + + with av.open(str(path), "w") as container: + stream = container.add_stream("libmp3lame", rate=RATE) + stream.layout = "mono" + + for _ in range(0, length, MIX_SAMPLES): + count = min(MIX_SAMPLES, length - _) + caller = array("h") + bot = array("h") + + caller.frombytes(self._caller.read(count * 2).ljust(count * 2, b"\0")) + bot.frombytes(self._bot.read(count * 2).ljust(count * 2, b"\0")) + + mixed = array("h", (max(-32768, min(32767, a + b)) for a, b in zip(caller, bot))) + frame = av.AudioFrame(format="s16", layout="mono", samples=count) + + frame.planes[0].update(mixed.tobytes()) + frame.sample_rate = RATE + + for packet in stream.encode(frame): + container.mux(packet) + for packet in stream.encode(None): + container.mux(packet) diff --git a/whatsapp_chatbot_python/calls/runtime.py b/whatsapp_chatbot_python/calls/runtime.py new file mode 100644 index 0000000..8417b97 --- /dev/null +++ b/whatsapp_chatbot_python/calls/runtime.py @@ -0,0 +1,133 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from enum import StrEnum +from time import monotonic +from .models import CallState +import asyncio + + +class RuntimeEventType(StrEnum): + STATE = "state" + BRIDGE_READY = "bridge_ready" + BRIDGE_FAILED = "bridge_failed" + VOICE_FAILED = "voice_failed" + END_CALL = "end_call" + DISCONNECT = "disconnect" + SHUTDOWN = "shutdown" + + +class RuntimeTimer(StrEnum): + RING = "ring" + TALK = "talk" + BRIDGE = "bridge" + + +@dataclass(frozen=True, slots=True) +class RuntimeEvent: + type: RuntimeEventType + state: str | None = None + reason: str | None = None + error: Exception | None = None + permanent: bool = False + + @classmethod + def server_state(cls, state: str, reason: str | None = None) -> RuntimeEvent: + return cls(RuntimeEventType.STATE, state=state, reason=reason) + + @classmethod + def bridge_ready(cls) -> RuntimeEvent: + return cls(RuntimeEventType.BRIDGE_READY) + + @classmethod + def bridge_failed(cls, error: Exception) -> RuntimeEvent: + return cls(RuntimeEventType.BRIDGE_FAILED, error=error) + + @classmethod + def voice_failed(cls, error: Exception) -> RuntimeEvent: + return cls(RuntimeEventType.VOICE_FAILED, error=error) + + @classmethod + def end_call(cls, reason: str | None = None) -> RuntimeEvent: + return cls(RuntimeEventType.END_CALL, reason=reason) + + @classmethod + def disconnect(cls, permanent: bool) -> RuntimeEvent: + return cls(RuntimeEventType.DISCONNECT, permanent=permanent) + + @classmethod + def shutdown(cls) -> RuntimeEvent: + return cls(RuntimeEventType.SHUTDOWN) + + +@dataclass(slots=True) +class CallRuntime: + ring_timeout_seconds: float + talk_timeout_seconds: float + bridge_timeout_seconds: float + events: asyncio.Queue[RuntimeEvent] = field(default_factory=asyncio.Queue) + ring_deadline: float | None = None + talk_deadline: float | None = None + bridge_deadline: float | None = None + + def start_ringing(self) -> None: + self.ring_deadline = monotonic() + self.ring_timeout_seconds + + def remote_accepted(self, bridge_ready: bool) -> None: + if self.talk_deadline is None: + self.talk_deadline = monotonic() + self.talk_timeout_seconds + + if not bridge_ready and self.bridge_deadline is None: + self.bridge_deadline = monotonic() + self.bridge_timeout_seconds + + def bridge_negotiated(self) -> None: + self.bridge_deadline = None + + async def next_event(self, state: CallState) -> RuntimeEvent | None: + try: + return self.events.get_nowait() + except asyncio.QueueEmpty: + pass + + timeout = self._next_timeout(state) + + try: + return await asyncio.wait_for(self.events.get(), timeout=timeout) + except TimeoutError: + return None + + def expired_timer(self, state: CallState) -> RuntimeTimer | None: + now = monotonic() + + if state == CallState.RINGING and self._expired(self.ring_deadline, now): + return RuntimeTimer.RING + + if state in {CallState.CONNECTING, CallState.IN_CALL}: + if self._expired(self.talk_deadline, now): + return RuntimeTimer.TALK + + if state == CallState.CONNECTING and self._expired(self.bridge_deadline, now): + return RuntimeTimer.BRIDGE + + return None + + def _next_timeout(self, state: CallState) -> float | None: + deadlines: list[float] = [] + + if state == CallState.RINGING and self.ring_deadline is not None: + deadlines.append(self.ring_deadline) + + if state in {CallState.CONNECTING, CallState.IN_CALL}: + if self.talk_deadline is not None: + deadlines.append(self.talk_deadline) + + if state == CallState.CONNECTING and self.bridge_deadline is not None: + deadlines.append(self.bridge_deadline) + + if not deadlines: + return None + + return max(0.0, min(deadlines) - monotonic()) + + @staticmethod + def _expired(deadline: float | None, now: float) -> bool: + return deadline is not None and now >= deadline diff --git a/whatsapp_chatbot_python/calls/service.py b/whatsapp_chatbot_python/calls/service.py new file mode 100644 index 0000000..c52f143 --- /dev/null +++ b/whatsapp_chatbot_python/calls/service.py @@ -0,0 +1,330 @@ +from __future__ import annotations +from contextlib import suppress +from whatsapp_api_client_python.API import GreenAPI +from whatsapp_api_client_python.tools.voip import CallAudio +from .contracts import TransitionCallback +from .realtime_voice import VoiceBotSession +from .recording import CallRecorder +from .models import CallEvent, CallExecutionResult, CallSession, CallState, TERMINAL_STATES +from .runtime import CallRuntime, RuntimeEvent, RuntimeEventType, RuntimeTimer +import asyncio +import logging + +# Constants + +BRIDGE_NEGOTIATION_TIMEOUT_SECONDS = 15 + +INITIAL_STATE_TIMEOUT_SECONDS = 10 + + +class WhatsAppCallService: + def __init__( + self, + *, + api_url: str, + id_instance: str, + api_token_instance: str, + openai_api_key: str, + realtime_model: str, + realtime_voice: str, + ring_timeout_seconds: int, + talk_timeout_seconds: int, + shutdown_timeout_seconds: int, + logger: logging.Logger, + ) -> None: + self._api_url = api_url + self._id_instance = id_instance + self._api_token_instance = api_token_instance + self._openai_api_key = openai_api_key + self._realtime_model = realtime_model + self._realtime_voice = realtime_voice + self._ring_timeout = ring_timeout_seconds + self._talk_timeout = talk_timeout_seconds + self._shutdown_timeout = shutdown_timeout_seconds + self._logger = logger + self._active_loop: asyncio.AbstractEventLoop | None = None + self._active_runtime: CallRuntime | None = None + self._stop_requested = False + + def request_stop(self) -> None: + self._stop_requested = True + loop = self._active_loop + runtime = self._active_runtime + + if loop is not None and runtime is not None and loop.is_running(): + loop.call_soon_threadsafe(runtime.events.put_nowait, RuntimeEvent.shutdown()) + + async def execute( + self, + call_session: CallSession, + transition: TransitionCallback, + ) -> CallExecutionResult: + if self._stop_requested: + transition(call_session, CallEvent.SHUTDOWN_REQUESTED, error_code="shutdown") + return CallExecutionResult() + + recorder = CallRecorder(self._logger) + + runtime = CallRuntime( + ring_timeout_seconds=self._ring_timeout, + talk_timeout_seconds=self._talk_timeout, + bridge_timeout_seconds=BRIDGE_NEGOTIATION_TIMEOUT_SECONDS, + ) + + self._active_loop = asyncio.get_running_loop() + self._active_runtime = runtime + + if self._stop_requested: + runtime.events.put_nowait(RuntimeEvent.shutdown()) + + voice = VoiceBotSession( + api_key=self._openai_api_key, + model=self._realtime_model, + voice=self._realtime_voice, + language=call_session.language, + on_error=lambda error: runtime.events.put_nowait(RuntimeEvent.voice_failed(error)), + recorder=recorder, + ) + + async def make_audio_session() -> CallAudio: + track = await voice.new_output_track() + + try: + sink = voice.new_input_sink() + except BaseException: + track.stop() + raise + + async def close_audio() -> None: + try: + await sink.close() + finally: + track.stop() + + return CallAudio(track, sink.attach, close_audio) + + try: + await voice.start() + api = GreenAPI(self._id_instance, self._api_token_instance, host=self._api_url) + calls = api.voip.connect(audio_factory=make_audio_session) + except Exception: + await voice.close() + recorder.finish() + self._active_runtime = None + self._active_loop = None + + raise + + bridge_task: asyncio.Task[None] | None = None + dialed = False + initial_state: asyncio.Future[str] = asyncio.get_running_loop().create_future() + dial_started = False + + def on_state(state) -> None: + if not initial_state.done(): + initial_state.set_result(state.state) + + if dial_started: + runtime.events.put_nowait(RuntimeEvent.server_state(state.state, state.reason)) + + def on_end_call(detail) -> None: + # NOTE: If the server returned the reason, it is a handled case. + # Otherwise it falls into the error handler. + runtime.events.put_nowait(RuntimeEvent.end_call(detail.get("cause", None))) + + def on_disconnect(detail) -> None: + permanent = bool(detail.get("permanent")) + + if permanent and not initial_state.done(): + initial_state.set_exception(RuntimeError("callsRtc connection refused")) + + runtime.events.put_nowait(RuntimeEvent.disconnect(permanent)) + + def on_bridge_done(task: asyncio.Task[None]) -> None: + if task.cancelled(): + return + + error = task.exception() + result = RuntimeEvent.bridge_failed(error) if error else RuntimeEvent.bridge_ready() + + runtime.events.put_nowait(result) + + calls.on("state", on_state) + calls.on("end_call", on_end_call) + calls.on("disconnect", on_disconnect) + + calls.on( + "error", + lambda detail: runtime.events.put_nowait( + RuntimeEvent.bridge_failed(RuntimeError("callsRtc reported an error")) + ), + ) + + try: + await calls.openAsync(timeout=INITIAL_STATE_TIMEOUT_SECONDS) + + state = await asyncio.wait_for(initial_state, timeout=INITIAL_STATE_TIMEOUT_SECONDS) + + if state != "idle": + raise RuntimeError(f"Instance already has a call: {state}") + + if self._stop_requested: + transition(call_session, CallEvent.SHUTDOWN_REQUESTED, error_code="shutdown") + else: + dial_started = True + + await api.voip.dialAsync(call_session.chat_id) + + dialed = True + + transition(call_session, CallEvent.DIAL_ACCEPTED) + runtime.start_ringing() + + # The library owns signaling and WebRTC; the voice session supplies audio. + # Bridge completion confirms the SDP answer, not ICE/DTLS or live audio. + bridge_task = asyncio.create_task(calls.startAudioAsync()) + bridge_task.add_done_callback(on_bridge_done) + + while call_session.state not in TERMINAL_STATES: + event = await runtime.next_event(call_session.state) + + if event is None: + await self._handle_timeout(runtime, api, call_session, transition) + else: + await self._handle_event(event, runtime, api, voice, recorder, call_session, transition) + except Exception as error: + self._logger.error( + "VoIP execution failed: session=%s error=%s", + call_session.session_id, + type(error).__name__, + ) + + transition( + call_session, CallEvent.INTERNAL_ERROR, + error_code=type(error).__name__, + ) + + if dialed: + await self._safe_hang_up(api) + finally: + with suppress(Exception): + await asyncio.wait_for(calls.closeAsync(), timeout=self._shutdown_timeout) + + with suppress(Exception): + await asyncio.wait_for(voice.close(), timeout=self._shutdown_timeout) + + if bridge_task is not None: + if not bridge_task.done(): + bridge_task.cancel() + + with suppress(Exception): + await asyncio.wait_for( + asyncio.gather(bridge_task, return_exceptions=True), + timeout=self._shutdown_timeout, + ) + + self._active_runtime = None + self._active_loop = None + + # Encoding runs after the RTC and Realtime connections have closed. + path = recorder.finish() + + if call_session.state not in {CallState.REMOTE_ENDED, CallState.TALK_TIMEOUT}: + # Preserve the established policy: discard partial failed calls. + if path is not None: + path.unlink(missing_ok=True) + + path = None + + return CallExecutionResult(path) + + async def _handle_event( + self, + event: RuntimeEvent, + runtime: CallRuntime, + api: GreenAPI, + voice: VoiceBotSession, + recorder: CallRecorder, + call_session: CallSession, + transition: TransitionCallback, + ) -> None: + if event.type == RuntimeEventType.STATE: + if event.state == "on-call": + transition(call_session, CallEvent.REMOTE_ACCEPTED) + runtime.remote_accepted(call_session.bridge_ready) + + if call_session.state == CallState.IN_CALL: + recorder.start() + await voice.greet_once() + elif event.state == "idle": + transition(call_session, CallEvent.REMOTE_IDLE, remote_reason=event.reason) + elif event.type == RuntimeEventType.BRIDGE_READY: + transition(call_session, CallEvent.BRIDGE_READY) + runtime.bridge_negotiated() + + if call_session.state == CallState.IN_CALL: + recorder.start() + await voice.greet_once() + elif event.type == RuntimeEventType.END_CALL: + if event.reason is None: + # The library reports connection-lost without a server cause. + raise RuntimeError("callsRtc connection lost before audio bridge") + + transition(call_session, CallEvent.REMOTE_IDLE, remote_reason=event.reason) + elif event.type == RuntimeEventType.BRIDGE_FAILED: + raise event.error or RuntimeError("Audio bridge failed") + elif event.type == RuntimeEventType.VOICE_FAILED: + raise event.error or RuntimeError("OpenAI Realtime failed") + elif event.type == RuntimeEventType.DISCONNECT: + if event.permanent: + raise RuntimeError("callsRtc connection permanently closed") + # The library reconnects and rebuilds an active audio bridge. + elif event.type == RuntimeEventType.SHUTDOWN: + transition(call_session, CallEvent.SHUTDOWN_REQUESTED, error_code="shutdown") + await self._safe_hang_up(api) + + async def _handle_timeout( + self, + runtime: CallRuntime, + api: GreenAPI, + call_session: CallSession, + transition: TransitionCallback, + ) -> None: + timer = runtime.expired_timer(call_session.state) + + if timer == RuntimeTimer.RING: + transition(call_session, CallEvent.RING_TIMER_EXPIRED) + await self._hang_up(api, call_session, transition) + elif timer == RuntimeTimer.TALK: + transition(call_session, CallEvent.TALK_TIMER_EXPIRED) + await self._hang_up(api, call_session, transition) + elif timer == RuntimeTimer.BRIDGE: + transition( + call_session, CallEvent.INTERNAL_ERROR, + error_code="bridge_negotiation_timeout", + ) + + await self._safe_hang_up(api) + + @staticmethod + async def _hang_up( + api: GreenAPI, + call_session: CallSession, + transition: TransitionCallback, + ) -> None: + try: + await api.voip.hangUpAsync() + except Exception as error: + transition( + call_session, CallEvent.INTERNAL_ERROR, + error_code=f"hangup_{type(error).__name__}", + ) + + return + + transition(call_session, CallEvent.HANGUP_CONFIRMED) + + @staticmethod + async def _safe_hang_up(api: GreenAPI) -> None: + with suppress(Exception): + await api.voip.hangUpAsync() diff --git a/whatsapp_chatbot_python/calls/state_machine.py b/whatsapp_chatbot_python/calls/state_machine.py new file mode 100644 index 0000000..91fc7d4 --- /dev/null +++ b/whatsapp_chatbot_python/calls/state_machine.py @@ -0,0 +1,194 @@ +from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass + +from .models import ( + CallEndReason, + CallEvent, + CallSession, + CallState, + StateTransition, + TERMINAL_STATES, + TransitionOutcome, + utc_now, +) + + +TransitionTarget = CallState | Callable[[CallSession], CallState] + + +@dataclass(frozen=True, slots=True) +class TransitionRule: + target: TransitionTarget + + def resolve(self, session: CallSession) -> CallState: + if callable(self.target): + return self.target(session) + + return self.target + + +def _bridge_ready_while_ringing(session: CallSession) -> CallState: + if session.remote_accepted: + return CallState.IN_CALL + + return CallState.RINGING + + +def _remote_accepted_while_ringing(session: CallSession) -> CallState: + if session.bridge_ready: + return CallState.IN_CALL + + return CallState.CONNECTING + + +def _ending_target(session: CallSession) -> CallState: + return { + CallEndReason.RING_TIMEOUT: CallState.RING_TIMEOUT, + CallEndReason.TALK_TIMEOUT: CallState.TALK_TIMEOUT, + }.get(session.end_reason, CallState.FAILED) + + +_TRANSITION_RULES: dict[tuple[CallState, CallEvent], TransitionRule] = { + (CallState.TRIGGERED, CallEvent.ENQUEUED): TransitionRule(CallState.QUEUED), + (CallState.QUEUED, CallEvent.DEQUEUED): TransitionRule(CallState.DIALING), + (CallState.QUEUED, CallEvent.CANCEL_REQUESTED): TransitionRule(CallState.CANCELLED), + (CallState.DIALING, CallEvent.DIAL_ACCEPTED): TransitionRule(CallState.RINGING), + (CallState.DIALING, CallEvent.REMOTE_IDLE): TransitionRule(CallState.REJECTED), + + (CallState.RINGING, CallEvent.BRIDGE_READY): TransitionRule( + _bridge_ready_while_ringing + ), + + (CallState.RINGING, CallEvent.REMOTE_ACCEPTED): TransitionRule( + _remote_accepted_while_ringing + ), + + (CallState.RINGING, CallEvent.REMOTE_IDLE): TransitionRule(CallState.REJECTED), + + (CallState.RINGING, CallEvent.RING_TIMER_EXPIRED): TransitionRule( + CallState.ENDING + ), + + (CallState.CONNECTING, CallEvent.BRIDGE_READY): TransitionRule(CallState.IN_CALL), + + # The remote party already answered; report a completed call even if the + # bridge never became ready. No recording is created until IN_CALL. + (CallState.CONNECTING, CallEvent.REMOTE_IDLE): TransitionRule( + CallState.REMOTE_ENDED + ), + + (CallState.CONNECTING, CallEvent.TALK_TIMER_EXPIRED): TransitionRule( + CallState.ENDING + ), + + (CallState.IN_CALL, CallEvent.REMOTE_IDLE): TransitionRule(CallState.REMOTE_ENDED), + (CallState.IN_CALL, CallEvent.TALK_TIMER_EXPIRED): TransitionRule(CallState.ENDING), + (CallState.ENDING, CallEvent.HANGUP_CONFIRMED): TransitionRule(_ending_target), + (CallState.ENDING, CallEvent.REMOTE_IDLE): TransitionRule(_ending_target), +} + + +for nonterminal_state in CallState: + if nonterminal_state in TERMINAL_STATES: + continue + + for failure_event in ( + CallEvent.INTERNAL_ERROR, + CallEvent.SHUTDOWN_REQUESTED, + ): + _TRANSITION_RULES[(nonterminal_state, failure_event)] = TransitionRule( + CallState.FAILED + ) + + +class CallStateMachine: + """ + Apply the formal call transition table and its associated session data. + + Duplicate, late and out-of-order events are intentionally ignored. An applied + event may update session data without changing state, for example an early + BRIDGE_READY event received while the remote side is still ringing. + """ + + def apply( + self, + session: CallSession, + event: CallEvent, + *, + remote_reason: str | None = None, + error_code: str | None = None, + ) -> StateTransition: + previous = session.state + + if previous in TERMINAL_STATES: + return self._ignored(previous, event) + + rule = _TRANSITION_RULES.get((previous, event)) + + if rule is None: + return self._ignored(previous, event) + + if remote_reason is not None: + session.remote_reason = remote_reason + + if error_code is not None: + session.error_code = error_code + + self._apply_event_data(session, event) + + target = rule.resolve(session) + + session.state = target + + if target in TERMINAL_STATES: + session.ended_at = utc_now() + + return StateTransition( + previous=previous, + current=target, + event=event, + outcome=TransitionOutcome.APPLIED, + ) + + @staticmethod + def _ignored(previous: CallState, event: CallEvent) -> StateTransition: + return StateTransition( + previous=previous, + current=previous, + event=event, + outcome=TransitionOutcome.IGNORED, + ) + + @staticmethod + def _apply_event_data(session: CallSession, event: CallEvent) -> None: + now = utc_now() + + if event == CallEvent.ENQUEUED: + session.queued_at = now + elif event == CallEvent.DEQUEUED: + session.dialing_at = now + elif event == CallEvent.CANCEL_REQUESTED: + session.end_reason = CallEndReason.USER_CANCELLED + elif event == CallEvent.REMOTE_ACCEPTED: + session.remote_accepted = True + session.answered_at = session.answered_at or now + elif event == CallEvent.BRIDGE_READY: + session.bridge_ready = True + elif event == CallEvent.RING_TIMER_EXPIRED: + session.end_reason = CallEndReason.RING_TIMEOUT + elif event == CallEvent.TALK_TIMER_EXPIRED: + session.end_reason = CallEndReason.TALK_TIMEOUT + elif event == CallEvent.REMOTE_IDLE: + CallStateMachine._apply_remote_idle(session) + elif event == CallEvent.SHUTDOWN_REQUESTED: + session.end_reason = CallEndReason.SHUTDOWN + elif event == CallEvent.INTERNAL_ERROR: + session.end_reason = CallEndReason.ERROR + + @staticmethod + def _apply_remote_idle(session: CallSession) -> None: + if session.state in {CallState.DIALING, CallState.RINGING}: + session.end_reason = CallEndReason.REMOTE_REJECTED + elif session.state in {CallState.CONNECTING, CallState.IN_CALL}: + session.end_reason = CallEndReason.REMOTE_HANGUP From 84145b4c29f7d265cc64aa85ecabf927fd7899b1 Mon Sep 17 00:00:00 2001 From: kennyromanov Date: Wed, 7 Oct 2026 15:28:16 +0500 Subject: [PATCH 2/4] chore: THe linting and the documentation have been updated --- .github/workflows/python-package.yml | 7 +- AGENTS.md | 1482 -------------------------- README.md | 79 +- docs/README.md | 82 +- ruff.toml | 6 + 5 files changed, 18 insertions(+), 1638 deletions(-) delete mode 100644 AGENTS.md create mode 100644 ruff.toml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9f90780..83e50ee 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,16 +29,13 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with ruff run: | - # stop the build if there are Python syntax errors or undefined names - ruff check --output-format=github --select=E9,F63,F7,F82 --target-version=py39 . - # default set of ruff rules with GitHub Annotations - ruff check --output-format=github --target-version=py39 . + ruff check --config ruff.toml --output-format=github . - name: Test with pytest run: | pytest tests --ignore=tests/calls calls: - name: VoIP / ${{ matrix.python-version }} + name: ${{ matrix.python-version }} (VoIP) runs-on: ubuntu-24.04 strategy: matrix: diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 9b53d24..0000000 --- a/AGENTS.md +++ /dev/null @@ -1,1482 +0,0 @@ -# sw-core - -Instructions for AI agents working with the repository. - ---- - -## 0) Quick facts about this repository - -* Purpose: a **runtime worker core** for managing live messenger instances through a custom WhatsApp server client. -* Current role: a lower-level runtime/service layer around `@green-api/sw-baileys`, responsible for live socket sessions, auth/session material, outbound commands, inbound events, webhook normalization, and local history. -* Current shape: - * Maintains local `WaInstance` runtimes - * Creates and supervises messenger sockets - * Stores auth/session state in Mongo - * Receives outbound commands through RabbitMQ and legacy REST paths - * Sends messages through live socket/runtime methods - * Receives messenger events from socket listeners - * Normalizes incoming/outgoing events into webhook/journal formats - * Publishes downstream webhook events through RabbitMQ - * Persists normalized message/chat/contact history in Mongo -* Stack: - * `TypeScript` - * `NestJS` - * `MongoDB` - * `RabbitMQ` - * Redis-like cache through `CacheFactory` - * `@green-api/sw-baileys` - * Custom Bayles/Baileys socket protocol -* Repository character: - * Legacy-heavy codebase - * Long-lived runtime sessions - * Stateful socket lifecycle - * Many explicit and implicit contracts - * Several externally observed behaviors that must not change casually - * Runtime behavior distributed across services, socket listeners, queues, caches, and persistence -* Key principle: this repository is **not** a clean standalone public API product, not a generic gateway, and not a place for broad architectural experiments. -* Architectural reality: many important contracts are implicit and historical. Queue names, message IDs, state changes, ACK/NACK behavior, retry behavior, webhook payloads, logs, caches, auth storage shape, and timing may be depended on by other services. -* Current core areas: - * `apps/swcore/src/app/wapi/wainstances/wa.instances.service.ts` — registry/supervisor of local Wa instances, lifecycle orchestration, outbound command bridge, reboot/delete flows - * `apps/swcore/src/app/wapi/wainstances/client/wa.instance.bayles.client.ts` — runtime of one Bayles instance: socket, auth state, reconnect, QR/pairing, event listeners, send methods, local caches - * `apps/swcore/src/app/wapi/controller/wa.controller.ts` — legacy REST surface for send, auth, QR, history, media, and instance operations - * `apps/swcore/src/app/wapi/wainstances/webhooks/webhook.service.bayles.ts` — incoming/outgoing event normalization, webhook queue publishing, journal/history mapping, dedupe/filter behavior - * `apps/swcore/src/app/wapi/wainstances/mapper/*` — webhook and journal mapper tree - * `apps/swcore/src/app/mongoauthstorage/authstore/use-multi-mongo-auth-state.ts` — Baileys auth state integration - * `apps/swcore/src/app/mongoauthstorage/authstore/auth.data/db/auth.data.mongo.service.ts` — auth data facade and storage routing - * `apps/swcore/src/app/wapi/rmq/*` — RabbitMQ topology, consumers, publishers, downloader integration - * `apps/swcore/src/app/wapi/jobs/jobs.ts` — periodic lifecycle/consumer jobs - * `apps/swcore/src/app/wapi/ws/qr.websocket.gateway.ts` — QR websocket gateway/cache behavior - -> Agents: before making changes, read the exact files involved in the affected flow. For lifecycle, outbound send, auth, webhook, or RabbitMQ changes, do not rely on one file only. Follow the runtime path end-to-end. - ---- - -## 1) Agent protocol - -1. **Understand the task first:** - * Determine whether the task changes: - * Instance lifecycle - * Socket/runtime behavior - * Authorization / QR / pairing flow - * Outbound send behavior - * RabbitMQ command handling - * REST legacy behavior - * Incoming event handling - * Webhook/journal normalization - * Mongo history persistence - * Auth/session storage - * Cache/reconnect behavior - * Media/downloader behavior - * Security/logging behavior - * Or only thin glue around existing behavior -2. **Assume legacy contracts exist:** - * If behavior is currently observed by external systems, treat it as a contract even when it is not documented. - * Do not change queue names, payload shapes, message ID semantics, status names, retry behavior, error behavior, delays, headers, or persistence shape without explicit intent. -3. **Trace the real runtime path:** - * For any change, identify where the flow starts and where it ends. - * Example: REST send starts in controller, goes through `WaInstancesService`, then into `WaInstanceBaylesClient`, then into Baileys socket, then later returns status through webhook events. - * Example: incoming message starts in socket listener, goes through `WebhookServiceBayles`, mapper tree, RabbitMQ publish, and Mongo history persistence. -4. **Minimal sufficient diff:** - * Modify only the files and sections required for the task. - * Minimal diff means the smallest complete safe change, including tests/docs/config when required. - * Do not perform broad cleanup, formatting sweeps, naming migrations, dependency upgrades, or framework reshaping unless explicitly required. -5. **Preserve runtime stability:** - * This service manages live sessions. A harmless-looking refactor can cause reconnect loops, lost events, duplicate webhooks, auth loss, delayed sends, or queue backlogs. -6. **Respect external dependencies:** - * SAPI/WaAdminJobs, RabbitMQ producers/consumers, Mongo collections, Media API, downloader service, Redis/cache, and downstream webhook consumers may depend on current behavior. -7. **Prefer explicit local fixes:** - * Add small named helpers when they clarify a concrete path. - * Avoid hiding runtime flow behind abstractions that make debugging harder. -8. **Avoid protocol invention:** - * Do not invent new lifecycle states, queue protocols, message ID formats, outbox semantics, delivery semantics, or auth storage categories unless explicitly requested. -9. **Document behavior changes:** - * If a real contract changes, update relevant docs, examples, configuration notes, and migration guidance. -10. **Be honest about uncertainty:** - * If a behavior appears legacy, implicit, or externally consumed, say so in the change summary. - * Do not present guesses as facts. - ---- - -## 2) Architecture and code placement - -### 2.1. Service shape - -This repository is best understood as: - -* A runtime-worker layer -* A local supervisor of live messenger instances -* A socket/session manager around Baileys -* A bridge between outbound command queues and live messenger sockets -* A normalizer between raw messenger events and downstream webhook/journal contracts -* A persistence layer for auth/session material and normalized history - -It is **not** best understood as: - -* A clean REST gateway -* A stateless controller-only service -* A generic public API product -* A full conversation platform -* A business workflow service -* A safe place for speculative architecture changes -* A repository where current behavior can be “cleaned up” without checking legacy consumers - -### 2.2. Current code placement - -**`apps/swcore/src/app/wapi/wainstances/wa.instances.service.ts`:** - -* Registry of local `WaInstanceBaylesClient` objects -* Instance lifecycle orchestration -* Startup initialization -* Reboot/spawn/delete flows -* Outbound command processing from RabbitMQ -* REST-to-runtime bridge methods -* Per-instance send mutex behavior -* Consumer start/stop coordination -* Some retry/failure handling around outbound messages - -**`apps/swcore/src/app/wapi/wainstances/client/wa.instance.bayles.client.ts`:** - -* Runtime for one messenger instance -* Baileys socket creation -* Connection lifecycle -* Reconnect behavior -* QR and pairing events -* Auth state usage -* Socket event listeners -* Send methods -* Local caches and reconnect slots -* State/status transitions -* Calls into webhook handling - -**`apps/swcore/src/app/wapi/controller/wa.controller.ts`:** - -* Legacy REST surface -* Direct runtime send path -* QR/pairing endpoints -* History endpoints -* Media/download endpoints -* Instance operation endpoints - -**`apps/swcore/src/app/wapi/wainstances/webhooks/webhook.service.bayles.ts`:** - -* Main incoming/outgoing event pipeline -* Webhook conversion -* Journal conversion -* Message status processing -* Dedupe-ish and filter behavior -* RabbitMQ webhook queue publishing -* History persistence coordination -* Event logging behavior - -**`apps/swcore/src/app/wapi/wainstances/mapper/interfaces.ts`:** - -* Public-ish enums and payload interfaces -* State/account/status/message/webhook types -* Many values here are externally observed and must be treated as contract-like - -**`apps/swcore/src/app/wapi/wainstances/mapper/webhook/*`:** - -* Webhook payload mapping -* Incoming/outgoing/call/avatar/state payload shapes -* Downstream compatibility concerns - -**`apps/swcore/src/app/wapi/wainstances/mapper/journal/*`:** - -* Journal/history mapping -* Mongo persistence compatibility concerns - -**`apps/swcore/src/app/mongoauthstorage/authstore/use-multi-mongo-auth-state.ts`:** - -* Integration point between Baileys auth state and Mongo auth storage -* `creds`, `keys.get`, `keys.set`, and `saveCreds` behavior -* Very sensitive protocol boundary - -**`apps/swcore/src/app/mongoauthstorage/authstore/auth.data/db/auth.data.mongo.service.ts`:** - -* Facade for auth data persistence -* Routes auth categories to sessions, prekeys, base categories, and creds services -* Storage shape compatibility matters - -**`apps/swcore/src/app/mongoauthstorage/model/*`:** - -* Mongo entities for auth/session material and normalized history -* Collection names and field shapes may be contract-like - -**`apps/swcore/src/app/wapi/rmq/topology/*`:** - -* Queue naming and topology -* Do not rename or reshape casually - -**`apps/swcore/src/app/wapi/rmq/rmq.consumer.service.ts`:** - -* Consumer lifecycle setup -* Instance-specific outgoing command consumption - -**`apps/swcore/src/app/wapi/rmq/rabbit/wa.message.consumer.ts`:** - -* RabbitMQ low-level consume behavior -* `prefetch`, ACK/NACK, and handler invocation - -**`apps/swcore/src/app/wapi/rmq/rmq.send.service.ts`:** - -* RabbitMQ publish path for downstream webhook events - -**`apps/swcore/src/app/wapi/rmq/rmq.downloader.service.ts`:** - -* Downloader queue integration for media events - -**`apps/swcore/src/app/wapi/ws/qr.websocket.gateway.ts`:** - -* QR gateway/cache/websocket delivery -* QR status compatibility concerns - -**`apps/swcore/src/app/wapi/jobs/jobs.ts`:** - -* Periodic jobs -* Consumer/reconnect/lifecycle behavior - -**`apps/swcore/src/app/common/factory/CacheFactory.ts`:** - -* Cache connection creation -* Sensitive logging risk -* Redis-like runtime dependency - -### 2.3. Main integration point - -The main integration point is **not** a controller method. It is the runtime relation between: - -```text -WaInstancesService - -> WaInstanceBaylesClient - -> @green-api/sw-baileys socket - -> socket events - -> WebhookServiceBayles - -> mapper tree - -> RabbitMQ / Mongo -``` - -Requirements: - -* Keep this flow traceable. -* Do not hide key lifecycle decisions behind generic infrastructure abstractions. -* Do not duplicate behavior that belongs to Baileys, the messenger protocol, or existing mapper logic. -* Preserve local runtime semantics unless explicitly changing them. - -### 2.4. Legacy reality - -This repository contains significant legacy behavior. - -Requirements: - -* Treat strange-looking code as potentially intentional until proven otherwise. -* Search for downstream usage before changing payloads, IDs, enums, statuses, queue names, or persistence shape. -* Avoid “cleanup” changes that alter timing, ordering, retries, logs, side effects, or fallback behavior. -* Do not replace legacy behavior with a cleaner model unless the task explicitly asks for a migration. -* If a change intentionally breaks legacy behavior, make that explicit in the summary and docs. - ---- - -## 3) Instance lifecycle and runtime state - -### 3.1. Core runtime objects - -The important runtime objects are: - -```typescript -export class WaInstanceBaylesClient extends EventEmitter { - public sock: WASocket | undefined = undefined - private statusInstance: StatusInstance = StatusInstance.Offline - private spawning = false - private attemptToConnect = 0 - private breakingDisconnectReason: BreakingDisconnectReason | undefined = undefined - private restartReason: number = undefined - private stateAccount: StateAccount -} -``` - -The service has two different state layers: - -```typescript -export enum StateAccount { - NotAuthorized = 'notAuthorized', - Authorized = 'authorized', - Blocked = 'blocked', - Starting = 'starting', - PendingCode = 'pendingCode', - YellowCard = 'yellowCard' -} - -export enum StatusInstance { - Online = 'online', - Offline = 'offline' -} -``` - -Requirements: - -* Do not merge `StateAccount` and `StatusInstance`. -* Do not assume `Authorized` means websocket is currently online. -* Do not assume `Online` means the account is semantically healthy. -* Preserve the distinction between auth/business state and live socket state. -* Treat state names as externally observed values. - -### 3.2. No clean FSM - -There are state enums, but there is no complete formal state machine. - -State transitions are distributed across: - -* `connection.update` -* `startAuthorization` -* `sendAuthorizationCode` -* `consumerJob` -* `logout` -* `resetSession` -* Reboot/spawn flows -* Reconnect handling -* Error branches - -Requirements: - -* Do not introduce a new FSM casually. -* Do not centralize state transitions as a cleanup unless explicitly requested and carefully tested. -* When changing state behavior, inspect all callers and downstream webhook behavior. -* State changes may emit webhooks, not just mutate local fields. - -### 3.3. Startup lifecycle - -At service startup: - -* `WaInstancesService.onModuleInit()` subscribes to SAPI updates and starts schedulers. -* Instance lists normally come from WaAdminJobs/SAPI. -* `initInstances()` creates local `WaInstanceBaylesClient` objects in chunks. -* Existing auth state in Mongo can trigger reboot/spawn behavior. -* Consumer startup may depend on authorization state. - -Requirements: - -* Do not assume all instances are present at process start. -* Do not assume local memory is the source of truth. -* Do not bypass SAPI/cache-api driven settings. -* Do not introduce pod-level ownership assumptions unless explicitly designed. -* Be careful with chunking, scheduling, and reboot behavior. - -### 3.4. Reconnect behavior - -Reconnect/supervision is not a single separate process. It is distributed across: - -* `connection.update` logic -* Periodic jobs -* Mutexes on reboot/connect/send -* Shared cache reconnect slots -* Local instance Map -* Status/account state transitions - -Requirements: - -* Do not remove or bypass reconnect guards because they appear redundant. -* Do not change reconnect delay/slot behavior without understanding multi-instance runtime effects. -* Do not add silent reconnect fallbacks that mask real auth/protocol failures. -* Do not assume there is a strong lease/heartbeat ownership model between pods unless you have verified it. - ---- - -## 4) Authorization, QR, pairing, and auth storage - -### 4.1. QR flow - -QR flow spans multiple layers: - -* REST QR endpoint -* QR websocket gateway/cache -* Service-level scan startup -* Baileys `connection.update` -* Socket event emission -* State/status changes - -Typical QR behavior: - -```typescript -if (qr) { - await this.changeStatusInstance(StatusInstance.Offline) - const qrImage = (await QRCode.toDataURL(qr)).replace('data:image/png;base64,', '') - this.emit('qr', { type: QRStatus.qrCode, message: qrImage }) - return -} -``` - -Requirements: - -* Do not change QR payload shape casually. -* Do not change QR cache/websocket behavior without checking REST and websocket consumers. -* Do not assume QR generation is independent of socket lifecycle. -* Preserve state/status side effects unless explicitly changing auth flow. - -### 4.2. Pairing code flow - -Pairing/mobile/phone-code flow uses separate endpoints and Baileys calls such as: - -```typescript -requestRegistrationCode(/* ... */) -register(/* ... */) -requestPairingCode(/* ... */) -``` - -Requirements: - -* Do not collapse QR and pairing code flow into one generic auth path without explicit design. -* Preserve existing mobile/phone-code semantics. -* Be careful with retry, state, and error behavior around pending code states. - -### 4.3. Successful authorization - -Successful authorization is detected through socket-level events such as: - -```typescript -connection === 'open' || isNewLogin -``` - -and then usually leads to: - -```typescript -await this.changeStatusInstance(StatusInstance.Online) -await this.changeStateAccount(StateAccount.Authorized) -await this.updateOnlineStatus() -``` - -Requirements: - -* Do not mark accounts authorized based only on REST request success. -* Do not bypass socket-confirmed auth transitions. -* Preserve webhook side effects of state changes. -* Be careful with `isNewLogin` behavior. - -### 4.4. Auth/session storage - -Auth/session/crypto state is stored through Mongo auth store integration. - -Important collections include: - -* `auth_creds` -* `auth_sessions` -* `auth_pre_keys` -* `auth_sender_keys` -* `auth_app_state_sync_keys` - -The Baileys auth state integration includes behavior like: - -```typescript -state: { - creds: authData.creds, - keys: { - get: async (category, ids) => ... - set: async (data) => ... - } -}, -saveCreds: async () => authDataService.saveAuthCreds(idInstance, authData.creds) -``` - -Requirements: - -* Treat auth storage as a protocol boundary. -* Do not change key categories, collection routing, serialization, or save timing casually. -* Do not rename auth collections or fields without migration planning. -* Do not add local encryption/decryption behavior without understanding infrastructure-level encryption and Baileys expectations. -* Do not log auth material, sessions, prekeys, sender keys, creds, or raw auth JSON. - ---- - -## 5) Outbound commands and send behavior - -### 5.1. There are two outbound paths - -There are two important send paths: - -1. **Legacy REST path** - * Controller calls runtime directly. - * It waits for runtime send promise. - * It usually returns `idMessage`. - * It is not an outbox API. - -2. **RabbitMQ command path** - * Commands arrive through `sw-outgoing-{id}`. - * Consumer parses `ProducerMessage`. - * Service dispatches by `producerMessage.type`. - * Runtime send uses `producerMessage.messageID`. - * ACK/NACK behavior controls retry/failure semantics. - -Requirements: - -* Do not merge REST and RabbitMQ semantics casually. -* Do not make REST send asynchronous/outbox-like unless explicitly requested. -* Do not remove RabbitMQ retry behavior when changing send methods. -* Do not assume send success means delivered/read. - -### 5.2. REST send semantics - -Typical REST path: - -```typescript -const idMessage = await this.waInstancesService.sendMessage(...) -res.status(HttpStatus.OK).json({ idMessage }) -``` - -Service path: - -```typescript -const waInstance = await this.throwWaInstance(idInstance) -return waInstance.sendMessage(...) -``` - -Runtime path eventually calls Baileys socket send methods. - -Requirements: - -* Preserve synchronous REST response semantics unless intentionally changing the public behavior. -* Preserve response shape such as `{ idMessage }` when existing clients depend on it. -* Do not add queue indirection to REST send as a cleanup. -* Do not report delivery/read status from REST send unless the protocol actually provides it at that time. - -### 5.3. RabbitMQ command semantics - -Typical command handling: - -```typescript -case TypesMessage.SendMessage: - await waInstance.sendMessage(..., producerMessage.messageID, ...) - break - -case TypesMessage.SendFileByUrl: - await waInstance.sendFileByUrl(..., producerMessage.messageID, ...) - break -``` - -Requirements: - -* Preserve queue names such as `sw-outgoing-{id}`. -* Preserve `ProducerMessage` shape unless explicitly changing producer/consumer contracts together. -* Preserve `producerMessage.messageID` semantics. -* Preserve ACK/NACK behavior. -* Preserve failure webhook behavior after retry exhaustion. -* Preserve per-instance send mutex unless explicitly redesigned. - -### 5.4. Message ID contracts - -Message IDs are not just arbitrary strings. - -Current behavior includes: - -* REST path may generate IDs internally. -* Some upload paths use `MessageIdService.generateID`. -* RabbitMQ path uses `producerMessage.messageID`. -* “Sent through API” classification may depend on structurally recognizable IDs. - -Requirements: - -* Do not change ID generation or parsing casually. -* Do not normalize IDs in ways that break downstream classification. -* Do not replace existing ID generation with UUIDs or random IDs unless explicitly required. -* Treat message ID structure as contract-like. - -### 5.5. Send success vs delivery status - -Send success means: - -* Runtime accepted/sent through socket -* A message ID was returned or reused -* It does **not** mean delivered -* It does **not** mean read -* Later status events produce webhook/status updates - -Requirements: - -* Do not rename send success into delivery success. -* Do not collapse send response and outgoing status webhook into one concept. -* Preserve separate outgoing status pipeline. - -### 5.6. Retry, throttling, and ordering - -RabbitMQ send path includes: - -* NACK for temporary errors -* ACK after final failure webhook -* Per-instance mutex -* Send delay through `delaySendMessagesMilliseconds` -* `MAX_DELAY_SEND_MESSAGES` -* Consumer `prefetch(1)` - -Requirements: - -* Treat retry and throttling as runtime behavior, not incidental implementation detail. -* Do not change `prefetch(1)` casually. -* Do not remove sleep/delay behavior as “unnecessary”. -* Do not convert per-instance serialized sends into parallel sends without explicit design. -* Be careful with timeout, socket-not-open, Mongo, uploader/downloader, and temporary infrastructure error branches. - ---- - -## 6) Incoming events, webhooks, journal, and history - -### 6.1. Socket listeners - -Runtime socket listeners are created after socket spawn and include events such as: - -```typescript -this.sock.ev.on('messages.upsert' /*, ...*/) -this.sock.ev.on('messages.update' /*, ...*/) -this.sock.ev.on('message-receipt.update' /*, ...*/) -this.sock.ev.on('presence.update' /*, ...*/) -this.sock.ws.on('CB:ack,class:message' /*, ...*/) -this.sock.ev.on('call_log' /*, ...*/) -``` - -Requirements: - -* Do not remove event listeners because they appear duplicate. -* Do not reorder event handling without understanding status/webhook effects. -* Do not assume Baileys events are clean, unique, or already normalized. -* Do not add expensive blocking work directly inside socket listeners. - -### 6.2. Main incoming pipeline - -Incoming messages roughly flow as: - -```text -messages.upsert - -> handleIncomingMessagesReceived(...) - -> convertToIncomingWebhooks(...) - -> mapper.webhook.incoming.process.toWebhook(...) - -> mapper.journal.incoming.process.toJournal(...) - -> addToQueueWithRetry(...) for webhook queue - -> repository upsert for history/journal -``` - -Requirements: - -* Preserve the separation between raw socket event, webhook payload, journal payload, queue publication, and history persistence. -* Do not change webhook and journal mappers independently if they must stay aligned. -* Do not remove dedupe/filter behavior without checking duplicate webhooks and history impact. -* Do not assume incoming processing is HTTP delivery. - -### 6.3. Webhook types - -Webhook types include values such as: - -* `incomingMessageReceived` -* `outgoingMessageStatus` -* `outgoingMessageReceived` -* `outgoingAPIMessageReceived` -* `stateInstanceChanged` -* `calls` -* `avatar` - -Requirements: - -* Treat webhook type names as external contracts. -* Do not rename, merge, or split webhook types casually. -* Do not add new webhook types without downstream compatibility planning. -* Preserve payload shape unless intentionally versioning or migrating consumers. - -### 6.4. Webhook delivery - -Webhook delivery is not direct HTTP from socket listeners. - -Current shape: - -```text -WebhookServiceBayles - -> RmqSendService - -> sw-incoming-{id} - -> downstream consumer(s) -``` - -Requirements: - -* Preserve queue-based webhook delivery unless explicitly changing architecture. -* Preserve queue naming conventions. -* Preserve retry publication behavior. -* Do not replace queue publish with direct HTTP delivery as a cleanup. -* Do not block socket event processing on slow downstream webhook consumers. - -### 6.5. Webhook publication retry - -Webhook queue publication uses retry behavior, currently with many attempts and long delay. - -Requirements: - -* Treat retry count/delay as operational behavior. -* Do not reduce retries casually. -* Do not silently drop webhook events. -* If adding failure handling, make it explicit and observable. - -### 6.6. History persistence - -Mongo collections include: - -* `ga_incoming_messages` -* `ga_outgoing_messages` -* `ga_chats` -* `ga_contacts` - -Requirements: - -* Treat collection names and document shapes as externally useful. -* Do not rename fields casually. -* Do not change upsert keys or dedupe semantics without checking history queries. -* Preserve history endpoints behavior when touching persistence. -* Be careful with migrations and backfills. - -### 6.7. History and polling APIs - -Current swcore has history endpoints such as: - -* `GetChatHistory` -* `lastIncomingMessages` -* `lastOutgoingMessages` - -A polling notification API such as `receiveNotification/deleteNotification` was not identified in this service. - -Requirements: - -* Do not invent notification polling behavior in swcore unless explicitly requested. -* Do not present history endpoints as webhook delivery. -* Keep history behavior separate from live event delivery. - ---- - -## 7) Media and downloader behavior - -### 7.1. Media event flow - -Media behavior may branch depending on configuration. - -When downloader integration is enabled: - -```text -media event - -> sw-downloads-{pool} - -> downloader service -``` - -There is also a lazy download endpoint that reads stored message data and uploads/downloads through Media API. - -Requirements: - -* Do not assume all media is downloaded immediately. -* Do not assume downloader service is always enabled. -* Do not bypass Media API integration casually. -* Preserve lazy download behavior unless intentionally changing media architecture. -* Do not load large media into memory without checking runtime impact. - -### 7.2. Media contracts - -Requirements: - -* Preserve media metadata shape in webhook/journal payloads. -* Preserve downloader queue naming. -* Preserve failure behavior when media is unavailable. -* Do not silently mark media downloaded if download/upload failed. -* Do not leak media URLs, tokens, or raw payloads in logs. - ---- - -## 8) Security, secrets, and sensitive data - -### 8.1. REST security boundary - -Many REST routes include `:apiTokenInstance`, but the middleware may be a legacy stub and may not enforce a real token check. - -Requirements: - -* Do not assume REST routes are securely protected by `apiTokenInstance`. -* Do not strengthen or change token validation casually because existing legacy clients may depend on current behavior. -* If asked to improve security, treat it as a contract and migration task, not a tiny middleware patch. -* Be explicit about compatibility impact. - -### 8.2. Sensitive data - -Sensitive data includes: - -* Auth creds -* Sessions -* Prekeys -* Sender keys -* App state sync keys -* Message bodies -* Media URLs -* Phone numbers / contacts -* Webhook payloads -* Redis/Mongo/Rabbit credentials -* SAPI/cache-api tokens -* Instance identifiers when combined with auth material - -Requirements: - -* Do not log sensitive material. -* Do not add debug logs with raw event payloads in production paths. -* Do not expose auth/session material through REST responses. -* Do not include secrets in error messages. -* Do not store secrets in new places without explicit design. - -### 8.3. Existing logging risks - -Known risky areas include: - -* Full event/webhook/journal logging under event logger flags -* Message body logging on send errors -* CacheFactory logging Redis password/user - -Requirements: - -* When touching these areas, reduce sensitive exposure if it is safe and compatible. -* Do not add new broad payload logs. -* Prefer structured, redacted logs. -* Preserve useful operational context without exposing secrets. - -Example preferred style: - -```typescript -this.logger.warn({ - idInstance, - messageId, - errorCode, - temporary: true, -}, 'outgoing command failed temporarily') -``` - -Avoid: - -```typescript -this.logger.warn({ - body: producerMessage, - authData, - redisPassword, -}, 'debug') -``` - ---- - -## 9) Cache, SAPI, and external configuration - -### 9.1. Cache behavior - -The service uses Redis-like cache through `CacheFactory`. - -Cache may affect: - -* Reconnect slots -* Per-instance prefixes -* Prekey TTL -* QR/cache delivery -* Runtime coordination -* SAPI/cache-api settings - -Requirements: - -* Do not treat cache as purely optional decoration. -* Do not change cache key prefixes casually. -* Do not change TTLs without checking runtime behavior. -* Do not log cache credentials. -* Do not replace cache logic with in-memory state unless explicitly designing single-process behavior. - -### 9.2. SAPI/cache-api settings - -Instance settings may come from SAPI/cache-api rather than local database tables. - -Requirements: - -* Do not add local settings as a competing source of truth. -* Do not bypass SAPI updates casually. -* Do not assume local memory has the latest instance configuration. -* Preserve subscription/update behavior at startup. - ---- - -## 10) Public and implicit contracts - -### 10.1. Contract types - -The public and implicit contracts include: - -* REST route paths -* REST request payloads -* REST response payloads -* REST status codes -* REST error behavior -* `apiTokenInstance` route shape -* RabbitMQ queue names -* RabbitMQ message shapes -* RabbitMQ ACK/NACK semantics -* Retry counts and retry delays -* Message ID format and classification -* Webhook event type names -* Webhook payload shape -* Journal/history payload shape -* Mongo collection names -* Mongo document fields -* Auth/session storage categories -* QR websocket payloads -* Status/state enum values -* Downloader queue names -* Media metadata shape -* Log formats consumed by operations -* Timing/order assumptions in consumers - -Requirements: - -* Treat contract changes as serious. -* If a change affects a contract, identify the consumers. -* If consumers are unknown, assume they exist. -* Preserve backward compatibility by default. -* Add compatibility aliases only when they do not create ambiguous behavior. -* Document intentional breaking changes. - -### 10.2. Explicit vs implicit behavior - -Requirements: - -* Do not assume undocumented means unused. -* Do not assume weird means wrong. -* Do not assume duplicate means redundant. -* Do not assume legacy means safe to remove. -* Do not assume a controller path is the only entrypoint. -* Do not assume a queue path has the same semantics as REST. - ---- - -## 11) Code style - -sw-core code should be **careful, explicit, and boring in a good way**. - -### 11.1. General principles - -* Prefer clear procedural flow over abstract indirection. -* Keep runtime paths readable end-to-end. -* Use early returns instead of deep nesting when safe. -* Add small named helpers only when they reduce local risk. -* Avoid speculative abstractions. -* Avoid broad formatting-only diffs. -* Keep stateful behavior visible. -* Preserve operational logs, but redact sensitive values. -* Prefer compatibility over elegance. - -### 11.2. Naming rules - -**Data → nouns:** - -```typescript -idInstance -producerMessage -messageId -payload -webhook -journal -authData -stateAccount -statusInstance -queueName -headers -metadata -``` - -**Actions → verbs:** - -```typescript -spawn -reboot -connect -logout -resetSession -sendMessage -sendFileByUrl -requestPairingCode -changeStateAccount -changeStatusInstance -handleIncomingMessagesReceived -convertToIncomingWebhooks -addToQueueWithRetry -buildOutgoingStatusWebhook -``` - -Requirements: - -* Use names that describe actual behavior. -* Do not use names that imply a cleaner or larger architecture than exists. -* Avoid vague infrastructure poetry such as `orchestrateFlow`, `processMagic`, `unifiedGateway`, `smartDispatcher`. -* Do not rename widely used legacy methods unless explicitly required. - -### 11.3. Function structure pattern - -Preferred local structure: - -1. **Doing some checks** — validate input, instance state, socket availability, required fields -2. **Getting the data** — parse payload, read runtime instance, derive options, fetch config -3. **Defining the functions** — only for small local helpers that clarify the current function -4. **Main flow** — call runtime/socket/mapper/repository/queue -5. **Handling result** — return ID/status/payload or ACK/NACK -6. **Handling errors** — classify temporary/permanent errors, preserve retry semantics - -Example: - -```typescript -async sendMessageFromCommand(producerMessage: ProducerMessage): Promise { - - // Doing some checks - - if (!producerMessage.idInstance) - throw new Error('idInstance is required'); - - - // Getting the data - - const waInstance = await this.throwWaInstance(producerMessage.idInstance); - - - await waInstance.sendMessage( - producerMessage.chatId, - producerMessage.message, - producerMessage.messageID, - ); -} -``` - -### 11.4. Comments - -Comments should explain **why**, not obvious syntax. - -Use these markers consistently when useful: - -```typescript -// Constants -// Variables -// Doing some checks -// Getting the data -// Defining the functions -// FIXME: -// TODO: -// NOTE: -``` - -Good comment: - -```typescript -// NOTE: Keep this check before ACK/NACK handling: temporary socket errors must be retried by RabbitMQ. -``` - -Bad comment: - -```typescript -counter++ // Increment counter by one -``` - ---- - -## 12) Runtime and infrastructure guidance - -### 12.1. Runtime reality - -Requirements: - -* Treat RabbitMQ, Mongo, Redis/cache, SAPI/cache-api, Media API, downloader, and Baileys as part of the working system. -* Do not validate behavior only by reading controller code. -* For socket lifecycle changes, inspect actual spawn/reconnect/auth/event paths. -* For queue changes, inspect producer message shape, consumer setup, ACK/NACK behavior, and downstream effects. -* For persistence changes, inspect query/read paths as well as writes. - -### 12.2. RabbitMQ guidance - -Requirements: - -* Preserve queue names unless explicitly changing topology. -* Preserve `prefetch(1)` unless intentionally changing send concurrency. -* Preserve ACK/NACK semantics. -* Preserve retry behavior for temporary errors. -* Preserve failed webhook behavior on final failure. -* Do not acknowledge failed commands before required failure side effects are completed. -* Do not NACK permanent errors forever. -* Do not publish malformed webhook payloads. - -### 12.3. Mongo guidance - -Requirements: - -* Treat auth collections as protocol-critical. -* Treat history collections as externally useful. -* Do not rename collections or fields without migration. -* Do not change upsert keys casually. -* Do not introduce unbounded writes in high-volume event paths. -* Be careful with indexes and query shapes. -* Do not store raw payloads in new collections without data-retention review. - -### 12.4. Baileys/socket guidance - -Requirements: - -* Do not reimplement messenger protocol behavior locally. -* Do not patch around SDK behavior with HTTP-layer hacks unless explicitly required. -* Do not swallow socket errors without observability. -* Do not change listener registration casually. -* Do not send messages when socket/account state says it is unsafe, unless existing behavior already does so and the task requires preserving it. -* Do not convert runtime state to stateless request/response logic. - -### 12.5. Multi-pod/process caution - -A clear lease/heartbeat ownership model for instances between pods may not exist. - -Requirements: - -* Do not add assumptions that only one pod can own an instance unless verified. -* Do not add local-only coordination for globally visible behavior. -* Be careful with shared cache reconnect slots. -* Be careful with duplicated consumers or duplicated socket sessions. -* If solving multi-pod ownership, treat it as an architecture task. - ---- - -## 13) API growth and change management - -### 13.1. Adding new behavior - -Good fits for this repository: - -* Small runtime-safe helpers around existing Baileys operations -* Explicit additions to existing send/auth/history/media flows -* Mapper extensions that preserve old payloads -* Operational fixes to reconnect/retry behavior -* Security hardening with compatibility planning -* Redacted logging improvements -* Bug fixes with narrow blast radius - -Poor fits unless explicitly required: - -* Full gateway redesign -* New public API product layer -* Generic outbox system -* Approval/workflow/business document logic -* New database ownership model -* New multi-pod lease system hidden inside a small bugfix -* Big FSM rewrite -* Queue topology migration without producer/consumer migration plan - -### 13.2. Backward compatibility - -When changing behavior: - -* Consider whether clients depend on current response shape, status code, error text, message ID, queue payload, webhook event type, or Mongo field. -* Preserve compatibility intentionally. -* Add new fields instead of changing existing fields when possible. -* Do not remove legacy aliases unless explicitly requested. -* If strict validation is added, treat it as a possible breaking change. -* If defaults are changed, explain why. - -### 13.3. Output behavior - -Requirements: - -* Keep webhook payload behavior predictable. -* Keep outgoing status behavior separate from send acceptance. -* Keep history writes aligned with webhook/journal mapping. -* Do not change media output shape casually. -* Do not change state/status event shape casually. -* Do not change ID generation casually. - ---- - -## 14) Domain-specific decision rules - -### 14.1. If the task touches lifecycle - -Read: - -* `wa.instances.service.ts` -* `wa.instance.bayles.client.ts` -* `jobs.ts` -* Auth store integration -* QR gateway if authorization is involved -* Rabbit consumer lifecycle if consumers are involved - -Ask: - -* Does this affect spawn/reboot/connect/logout/reset? -* Does this affect state/status webhooks? -* Does this affect existing sessions? -* Does this affect consumer start/stop? -* Does this affect reconnect behavior? - -Default posture: - -* Make the smallest safe change. -* Preserve existing state names and transitions. -* Avoid centralizing lifecycle logic unless explicitly required. - -### 14.2. If the task touches outbound send - -Read: - -* Controller send endpoint if REST is involved -* `WaInstancesService` send methods -* `WaInstanceBaylesClient` send methods -* RabbitMQ consumer if queue commands are involved -* Webhook status handling if status behavior is involved -* Message ID utility if IDs are involved - -Ask: - -* Is this REST path or RabbitMQ path? -* Who owns `messageID`? -* What happens on temporary error? -* What happens after retry exhaustion? -* Does success mean accepted or delivered? -* Does this affect rate delay or ordering? - -Default posture: - -* Preserve accepted-vs-delivered semantics. -* Preserve ACK/NACK behavior. -* Preserve per-instance serialization. - -### 14.3. If the task touches incoming events or webhooks - -Read: - -* Socket listener registration -* `WebhookServiceBayles` -* Mapper interfaces -* Webhook mapper tree -* Journal mapper tree -* RabbitMQ publisher -* History repositories/entities - -Ask: - -* Which raw event starts the flow? -* Which webhook type is emitted? -* Is history written? -* Is media involved? -* Is this incoming, outgoing, status, call, avatar, or state? -* Does dedupe/filter behavior change? - -Default posture: - -* Preserve payload shape. -* Preserve queue publication. -* Preserve history alignment. - -### 14.4. If the task touches auth/session storage - -Read: - -* `use-multi-mongo-auth-state.ts` -* `auth.data.mongo.service.ts` -* Auth models -* Baileys socket creation path -* Reboot/spawn path - -Ask: - -* Does this affect existing sessions? -* Does this affect key categories? -* Does this affect save timing? -* Does this affect serialization? -* Does this leak sensitive material? -* Is migration required? - -Default posture: - -* Treat as high-risk. -* Avoid changes unless necessary. -* Never log raw auth data. - -### 14.5. If the task touches security - -Read: - -* Middleware -* Controller route structure -* Instance account model -* Auth/session storage -* Logging paths -* Config/env usage - -Ask: - -* Is this changing a legacy behavior? -* Are existing clients relying on weak/no validation? -* Is rollout/migration needed? -* Are secrets or message bodies exposed? -* Can hardening be done with compatibility? - -Default posture: - -* Improve redaction safely. -* Do not silently break clients. -* Do not overstate current security guarantees. - -### 14.6. If the task touches media - -Read: - -* Webhook media handling -* Downloader service -* Lazy download endpoint -* Media API client -* Stored message entities - -Ask: - -* Is downloader enabled? -* Is this lazy or eager download? -* Is metadata shape changing? -* Are files or URLs sensitive? -* Does this affect memory usage? - -Default posture: - -* Preserve branch behavior. -* Avoid loading large files in memory. -* Preserve Media API contract. - ---- - -## 15) Documentation and verification - -### 15.1. Documentation responsibilities - -If functionality changes, update as needed: - -* Route/API docs -* Queue contract docs -* Webhook payload examples -* State/status documentation -* Auth/session migration notes -* Environment/config notes -* Operational runbooks -* Comments near public behavior -* Examples used by downstream services - -Requirements: - -* Do not let code and docs diverge further. -* If docs are already incomplete, add narrowly useful notes near the changed behavior. -* Do not document guessed behavior as certain. - -### 15.2. Verification priorities - -Because this repository is a runtime integration layer, useful checks are usually flow-based. - -Prefer: - -* Real or integration-style send path checks -* RabbitMQ command handler checks -* ACK/NACK behavior checks -* Webhook payload shape checks -* History persistence checks -* Auth storage compatibility checks -* QR/pairing flow checks -* Reconnect/reboot smoke checks -* Media downloader/lazy download checks when touched - -Less useful by itself: - -* Decorative unit tests around mocked helpers that do not exercise runtime semantics -* Snapshot tests that bless accidental legacy behavior without explanation -* Controller-only checks for socket/event behavior - -### 15.3. Change summary requirements - -When summarizing a change, include: - -* What flow was changed -* Which contracts were preserved -* Whether REST, RabbitMQ, webhook, Mongo, auth, or media behavior changed -* Whether retries/ACK/NACK/delays were changed -* Whether message ID or state/status semantics changed -* What was verified -* What remains risky or unverified - -Example summary: - -```text -Changed the RabbitMQ outbound SendFileByUrl handling to classify uploader timeout as a temporary error. - -Preserved: -- sw-outgoing-{id} queue contract -- producerMessage.messageID usage -- per-instance send mutex -- ACK after final failed webhook -- REST send behavior - -Verified: -- temporary timeout returns NACK -- permanent validation error is not retried -- failed webhook is emitted after retry exhaustion -``` - ---- - -## 16) Do Not - -* ❌ Don’t describe this repository as a clean standalone REST API product. -* ❌ Don’t treat REST controllers as the core of the system. -* ❌ Don’t ignore `WaInstancesService` and `WaInstanceBaylesClient` when touching runtime behavior. -* ❌ Don’t change live socket lifecycle casually. -* ❌ Don’t change auth/session storage shape casually. -* ❌ Don’t log auth material, session keys, prekeys, sender keys, message bodies, media URLs, or credentials. -* ❌ Don’t assume `apiTokenInstance` is a strong security boundary. -* ❌ Don’t silently strengthen legacy REST auth without a compatibility plan. -* ❌ Don’t merge `StateAccount` and `StatusInstance`. -* ❌ Don’t add, rename, or remove state/status enum values casually. -* ❌ Don’t introduce a new FSM as a cleanup. -* ❌ Don’t change QR or pairing behavior without tracing the full auth flow. -* ❌ Don’t convert REST send into an async outbox API unless explicitly required. -* ❌ Don’t treat send success as delivered/read success. -* ❌ Don’t change message ID generation or parsing casually. -* ❌ Don’t rename RabbitMQ queues casually. -* ❌ Don’t change `ProducerMessage` shape casually. -* ❌ Don’t remove `prefetch(1)` casually. -* ❌ Don’t bypass ACK/NACK retry semantics. -* ❌ Don’t remove per-instance send mutexes casually. -* ❌ Don’t remove send delays/rate throttling as “cleanup”. -* ❌ Don’t replace queue-based webhook delivery with direct HTTP delivery casually. -* ❌ Don’t change webhook type names casually. -* ❌ Don’t reshape webhook payloads without downstream compatibility planning. -* ❌ Don’t change journal/history mapping independently from webhook behavior when they must stay aligned. -* ❌ Don’t rename Mongo collections or fields without migration. -* ❌ Don’t change auth key category routing without understanding Baileys expectations. -* ❌ Don’t add broad abstractions that hide runtime flow. -* ❌ Don’t migrate frameworks or rewrite services for style. -* ❌ Don’t perform formatting sweeps mixed with behavior changes. -* ❌ Don’t remove legacy behavior just because it looks strange. -* ❌ Don’t assume undocumented behavior is unused. -* ❌ Don’t assume duplicate-looking logic is redundant. -* ❌ Don’t add silent fallbacks that hide protocol/runtime failures. -* ❌ Don’t add local-only state for globally visible multi-pod behavior. -* ❌ Don’t bypass SAPI/cache-api as source of instance settings. -* ❌ Don’t treat Redis/cache as a harmless optimization. -* ❌ Don’t add downloader/media behavior that loads large files into memory without need. -* ❌ Don’t expose raw errors from Baileys, Mongo, RabbitMQ, Redis, or Media API if they contain sensitive data. -* ❌ Don’t deepen divergence between implementation, docs, queues, webhook examples, and operational reality. -* ❌ Don’t turn this legacy runtime-worker into architecture theater. - ---- - -## 17) Pre-change checklist - -* [ ] The affected flow was identified: lifecycle, auth, outbound, inbound, webhook, history, media, security, cache, or runtime setup. -* [ ] The real runtime path was traced end-to-end. -* [ ] Legacy contracts were considered before changing behavior. -* [ ] Public and implicit contracts were preserved unless intentionally changed. -* [ ] Queue names and RabbitMQ ACK/NACK behavior were preserved unless intentionally changed. -* [ ] Message ID semantics were preserved unless intentionally changed. -* [ ] Send accepted vs delivered/read semantics were not confused. -* [ ] StateAccount and StatusInstance semantics were preserved. -* [ ] Auth/session storage shape and save behavior were not broken. -* [ ] Sensitive data was not logged or exposed. -* [ ] Webhook payload shape and event types were preserved unless intentionally changed. -* [ ] Journal/history persistence stayed aligned with webhook behavior. -* [ ] Cache/SAPI/runtime assumptions were checked. -* [ ] Minimal sufficient diff was used. -* [ ] No broad refactors, formatting sweeps, or speculative abstractions were introduced. -* [ ] Documentation/examples/config notes were updated if behavior changed. -* [ ] Verification matched the real integration flow, not only isolated helper code. -* [ ] Remaining risk or uncertainty was stated clearly. diff --git a/README.md b/README.md index 3d519a1..41113ed 100644 --- a/README.md +++ b/README.md @@ -529,15 +529,14 @@ optional dependencies (or use `'.[voip]'` when installing from this checkout): python -m pip install 'whatsapp-chatbot-python[voip]' ``` -The extra pins the SDK and audio/provider versions used by the source demo. -The ordinary chatbot import does not load OpenAI, aiortc or PyAV. Call models, -contracts and the FSM are available from `whatsapp_chatbot_python.calls`; -import the executor explicitly from `whatsapp_chatbot_python.calls.service`. +The extra pins the SDK and audio/provider versions used. The ordinary chatbot import does not load OpenAI, aiortc or +PyAV. Call models, contracts and the FSM are available from `whatsapp_chatbot_python.calls`; import the executor +explicitly from `whatsapp_chatbot_python.calls.service`. ### Execute one call -The model and voice are supplied by the application. This example performs an -actual call when run with valid credentials: +The model and voice are supplied by the application. This example performs an actual call when run with valid +credentials: ```python import asyncio @@ -584,74 +583,6 @@ async def call_once(): asyncio.run(call_once()) ``` -`execute(session, transition)` retains the demo's interface. Prepare the session -in `DIALING` (the `ENQUEUED` / `DEQUEUED` transitions above do this). The callback -must **synchronously apply each transition to the same session**, not merely -log it. A coordinator should apply transitions under its existing lock. -The final state, end reason and error code are on the mutated `CallSession`; -`CallExecutionResult` contains only `recording_path: Path | None`. - -Run calls serially on each service instance. `request_stop()` is thread-safe -and permanently requests shutdown of that executor; create a new executor to -resume. Initialization failures may propagate to the caller, as in the demo; -the application/coordinator must handle them. Once execution reaches its main -call loop, handled errors are reflected through FSM transitions. - -### Integrate with an existing demo coordinator - -Replace the old service import with: - -```python -from whatsapp_chatbot_python.calls.service import WhatsAppCallService -from whatsapp_chatbot_python.calls.contracts import CallExecutor, TransitionCallback -from whatsapp_chatbot_python.calls import CallSession, CallStateMachine -``` - -Keep the constructor arguments and the existing worker invocation: - -```python -result = asyncio.run(executor.execute(session, coordinator.transition)) -``` - -Queueing, duplicate prevention, chat commands, menu state, localized result -texts and uploading the recording remain in the application. Keep the call in -the coordinator's worker: waiting for it inside a synchronous message handler -blocks notification processing. The executor creates its own GreenAPI client; -no `Notification` or chatbot API object needs to cross threads. - -Old model/FSM imports can be retained with explicit re-exports from this package. -Keep `EnqueueResult` in the demo. Move test mock targets from -`internal.calls.service.*` to `whatsapp_chatbot_python.calls.service.*` (and -likewise for audio modules): re-exporting a class does not redirect its globals. - -### Audio and recording behavior - -The original audio pipeline is preserved: mono PCM16 at 24 kHz, 20 ms output -frames, caller interruption, bounded playback buffering, and replacement audio -tracks on reconnect while retaining one conversation and greeting. Greeting -starts after remote acceptance and audio bridge negotiation; negotiation does -not itself guarantee live ICE/DTLS audio. The talk timer starts at remote -acceptance, including time waiting for the bridge. - -Recording mixes both parties and keeps pauses. A recording is returned only -for `REMOTE_ENDED` or `TALK_TIMEOUT`, and may still be `None` if there was no -audio or recording failed. Failed-call partial recordings are discarded. -The returned temporary MP3 belongs to the caller: upload/copy it if needed and -always delete it afterwards. Recording failure does not by itself fail the call. - -### Tests - -```shell -python -m pip install -e '.[voip]' pytest -python -m pytest tests -``` - -The call tests cover dialing, acceptance/bridge ordering, reconnects, shutdown, -timeouts, playback interruption and real MP3 encode/decode. Network/provider -behavior is simulated. Without optional audio dependencies, pytest collects -only the lightweight FSM/runtime tests; on Python below 3.11 it omits the call -suite. CI installs the extra and runs the full suite on Python 3.11–3.13. - ## Service methods documentation [Service methods documentation](https://green-api.com/en/docs/api/) diff --git a/docs/README.md b/docs/README.md index 9f491f8..139425c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -515,23 +515,20 @@ bot.run_forever() ## Голосовые звонки (необязательное дополнение VoIP) -Пакет `whatsapp_chatbot_python.calls` выполняет один исходящий WhatsApp-звонок: -подключение через GreenAPI SDK, голосовой разговор с OpenAI Realtime, -тайм-ауты, воспроизведение и запись MP3. Требуются **Python 3.11+**, +Пакет `whatsapp_chatbot_python.calls` выполняет один исходящий WhatsApp-звонок: подключение через GreenAPI SDK, +голосовой разговор с OpenAI Realtime, тайм-ауты, воспроизведение и запись MP3. Требуются **Python 3.11+**, авторизованный инстанс с поддержкой VoIP и ключ OpenAI. ```shell python -m pip install 'whatsapp-chatbot-python[voip]' ``` -Для установки из исходников используйте `python -m pip install -e '.[voip]'`. -Дополнение фиксирует версии SDK, OpenAI и аудиобиблиотек из исходного демо. -Обычный импорт чат-бота не загружает OpenAI, aiortc или PyAV. +Для установки из исходников используйте `python -m pip install -e '.[voip]'`. Дополнение фиксирует версии SDK, OpenAI и +аудиобиблиотек. Обычный импорт чат-бота не загружает OpenAI, aiortc или PyAV. ### Выполнить один звонок -Модель и голос задаёт приложение. При запуске с действительными реквизитами -этот пример совершит настоящий звонок: +Модель и голос задаёт приложение. При запуске с действительными реквизитами этот пример совершит настоящий звонок: ```python import asyncio @@ -578,75 +575,6 @@ async def call_once(): asyncio.run(call_once()) ``` -Сохранён интерфейс `await service.execute(session, transition)`. -Перед выполнением сессия должна находиться в `DIALING`: выше её подготавливают -переходы `ENQUEUED` и `DEQUEUED`. Callback обязан синхронно применять FSM к -той же сессии. В координаторе это делается под его существующей блокировкой. -Callback, который только логирует событие, для этого контракта не подходит. - -Итоговое состояние, причина завершения и код ошибки находятся в изменённой -`CallSession`. `CallExecutionResult` возвращает только путь к записи либо `None`. -На одном исполнителе звонки выполняются последовательно. -`request_stop()` можно вызвать из другого потока: он навсегда запрашивает -остановку данного исполнителя. Для возобновления создайте новый объект. -Ошибки начальной подготовки могут выбрасываться вызывающей стороне; координатор -должен их обрабатывать, как в исходном демо. - -### Подключение к демо - -Замените импорт сервиса и общих контрактов: - -```python -from whatsapp_chatbot_python.calls.service import WhatsAppCallService -from whatsapp_chatbot_python.calls.contracts import CallExecutor, TransitionCallback -from whatsapp_chatbot_python.calls import CallSession, CallStateMachine -``` - -Аргументы конструктора сохраняются. В рабочем потоке координатора остаётся: - -```python -result = asyncio.run(executor.execute(session, coordinator.transition)) -``` - -Очередь, защита от повторных заявок, команды чата, меню, локализованные ответы -и загрузка записи остаются в приложении. Не ждите звонок внутри синхронного -обработчика сообщений: это задержит обработку остальных уведомлений. -Исполнитель создаёт собственный GreenAPI-клиент; передавать ему `Notification` -или API-клиент обработчика не требуется. - -Старые импорты моделей и FSM можно сохранить явными реэкспортами. -`EnqueueResult` остаётся в демо. Пути подмен в тестах нужно изменить с -`internal.calls.service.*` на `whatsapp_chatbot_python.calls.service.*`, -аналогично для аудиомодулей: реэкспорт класса не меняет его глобальные зависимости. - -### Аудио и запись - -Сохранены PCM16 mono 24 кГц, кадры по 20 мс, прерывание ответа собеседником, -ограничение буфера и замена аудиотреков при переподключении без нового разговора -и повторного приветствия. Приветствие начинается после ответа и согласования -моста; согласование само по себе не подтверждает наличие живого аудио ICE/DTLS. -Таймер разговора запускается при ответе собеседника, включая ожидание моста. - -Запись объединяет обе стороны и сохраняет паузы. Файл возвращается только при -`REMOTE_ENDED` и `TALK_TIMEOUT`; отсутствие аудио или ошибка записи могут дать -`None` и в этих случаях. Частичные записи неуспешных звонков удаляются. -Возвращённым временным MP3 владеет приложение: после отправки или копирования -оно обязано удалить файл, предпочтительно через `finally`. -Ошибка записи сама по себе не завершает разговор как неуспешный. - -### Проверка - -```shell -python -m pip install -e '.[voip]' pytest -python -m pytest tests -``` - -Тесты проверяют дозвон, порядок ответа и готовности моста, переподключения, -остановку, тайм-ауты, прерывания аудио и настоящее кодирование/декодирование MP3. -Сеть и провайдер моделируются. Без аудиозависимостей pytest собирает только -лёгкие тесты FSM и таймеров; на Python ниже 3.11 тесты звонков не собираются. -CI устанавливает дополнение и запускает полный набор на Python 3.11–3.13. - ## Документация по методам сервиса [Документация по методам сервиса](https://green-api.com/docs/api/) diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..46b0b15 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,6 @@ +# Match the SDK's lint policy: catch code errors without rewriting legacy style. +target-version = "py39" +force-exclude = true + +[lint] +select = ["E4", "E7", "E9", "F"] From 52b7728d68123265b4afeb7c59eda7f428a8392f Mon Sep 17 00:00:00 2001 From: kennyromanov Date: Wed, 7 Oct 2026 16:54:24 +0500 Subject: [PATCH 3/4] fix: The issue with the cleanup has been fixed; THe cleanup process has been improved --- setup.py | 9 +- tests/calls/test_call_cleanup.py | 315 +++++++++++++++++++ whatsapp_chatbot_python/calls/contracts.py | 2 +- whatsapp_chatbot_python/calls/service.py | 343 ++++++++++++++------- 4 files changed, 547 insertions(+), 122 deletions(-) create mode 100644 tests/calls/test_call_cleanup.py diff --git a/setup.py b/setup.py index d99b6ed..e87f056 100644 --- a/setup.py +++ b/setup.py @@ -46,10 +46,11 @@ install_requires=["whatsapp-api-client-python>=0.0.53"], extras_require={ "voip": [ - "whatsapp-api-client-python[voip]==0.0.55", - "openai==3.20.0", - "aiortc==1.15.0", - "av==17.1.0", + "whatsapp-api-client-python[voip]==0.0.55; python_version >= '3.11'", + "openai==3.20.0; python_version >= '3.11'", + "aiortc==1.15.0; python_version >= '3.11'", + "av==17.1.0; python_version >= '3.11'", + "websockets>=16,<17; python_version >= '3.11'", ], }, python_requires=">=3.7" diff --git a/tests/calls/test_call_cleanup.py b/tests/calls/test_call_cleanup.py new file mode 100644 index 0000000..50afb79 --- /dev/null +++ b/tests/calls/test_call_cleanup.py @@ -0,0 +1,315 @@ +import asyncio +import unittest +from unittest.mock import patch + +from whatsapp_chatbot_python.calls.models import CallEndReason, CallEvent, CallState +from whatsapp_chatbot_python.calls.recording import CallRecorder + +from . import test_call_service as fixtures + + +class CallCleanupTest(unittest.IsolatedAsyncioTestCase): + make_session = fixtures.CallServiceTest.make_session + make_service = fixtures.CallServiceTest.make_service + + def setUp(self): + fixtures.CallServiceTest.setUp(self) + self.recorders = [] + recorders = self.recorders + + class ObservedRecorder(CallRecorder): + def __init__(self, logger): + super().__init__(logger) + self.finish_calls = 0 + self.finished_path = None + recorders.append(self) + + def finish(self): + self.finish_calls += 1 + self.finished_path = super().finish() + return self.finished_path + + patcher = patch("whatsapp_chatbot_python.calls.service.CallRecorder", ObservedRecorder) + patcher.start() + self.addCleanup(patcher.stop) + + def assert_recorder_closed(self, service): + recorder = self.recorders[-1] + self.assertEqual(recorder.finish_calls, 1) + self.assertTrue(recorder._caller.closed) + self.assertTrue(recorder._bot.closed) + if recorder._thread is not None: + self.assertFalse(recorder._thread.is_alive()) + self.assertIsNone(service._active_loop) + self.assertIsNone(service._active_runtime) + + async def test_execute_accepts_contract_keyword_arguments(self): + session, transition = self.make_session() + service = self.make_service() + await service.execute(session=session, transition=transition) + self.assertEqual(session.state, CallState.REMOTE_ENDED) + self.assert_recorder_closed(service) + + async def test_startup_errors_finish_recorder_and_still_propagate(self): + for stage in ("constructor", "start", "connect", "listeners"): + with self.subTest(stage=stage): + session, transition = self.make_session() + service = self.make_service() + if stage == "constructor": + target = "whatsapp_chatbot_python.calls.service.VoiceBotSession" + elif stage == "start": + target = "tests.calls.test_call_service.FakeVoice.start" + elif stage == "connect": + target = "tests.calls.test_call_service.FakeClient.connect" + else: + target = "tests.calls.test_call_service.FakeCalls.on" + + with patch(target, side_effect=RuntimeError("startup failed")): + with self.assertRaisesRegex(RuntimeError, "startup failed"): + await service.execute(session, transition) + self.assert_recorder_closed(service) + + async def cancel_at_stage(self, stage, *, repeat=False, hangup_fails=False): + fixtures.FakeClient.answer = stage != "ring" + fixtures.FakeClient.remote_hangup = stage == "cleanup" + session, apply = self.make_session() + service = self.make_service(ring=5, talk=5) + reached = asyncio.Event() + close_entered = asyncio.Event() + release_close = asyncio.Event() + cancel_seen = asyncio.Event() + if stage != "cleanup" and not repeat: + release_close.set() + + start = fixtures.FakeVoice.start + open_connection = fixtures.FakeCalls.openAsync + dial = fixtures.FakeClient.dialAsync + bridge = fixtures.FakeCalls.startAudioAsync + greet = fixtures.FakeVoice.greet_once + close = fixtures.FakeCalls.closeAsync + hangup = fixtures.FakeClient.hangUpAsync + + async def pause(name): + if stage == name: + reached.set() + await asyncio.Event().wait() + + async def voice_start(voice): + await start(voice) + await pause("start") + + async def calls_open(calls, **options): + await open_connection(calls, **options) + await pause("open") + + async def calls_dial(client, target): + await dial(client, target) + await pause("dial") + + async def calls_bridge(calls): + await pause("bridge") + await bridge(calls) + + async def voice_greet(voice): + await greet(voice) + # Exercise the real recording thread and MP3 encoding on cancellation. + recorder = voice.options["recorder"] + recorder.set_generation(1) + recorder.add_bot(b"\x01\x00" * 480, 1) + await pause("talk") + + async def calls_close(calls): + close_entered.set() + if stage == "cleanup": + reached.set() + await release_close.wait() + await close(calls) + + async def calls_hangup(client): + await hangup(client) + if hangup_fails: + raise RuntimeError("hangup failed") + + def transition(call_session, event, **options): + result = apply(call_session, event, **options) + if stage == "ring" and event == CallEvent.DIAL_ACCEPTED: + reached.set() + if event == CallEvent.SHUTDOWN_REQUESTED: + cancel_seen.set() + return result + + with ( + patch.object(fixtures.FakeVoice, "start", voice_start), + patch.object(fixtures.FakeCalls, "openAsync", calls_open), + patch.object(fixtures.FakeClient, "dialAsync", calls_dial), + patch.object(fixtures.FakeCalls, "startAudioAsync", calls_bridge), + patch.object(fixtures.FakeVoice, "greet_once", voice_greet), + patch.object(fixtures.FakeCalls, "closeAsync", calls_close), + patch.object(fixtures.FakeClient, "hangUpAsync", calls_hangup), + ): + task = asyncio.create_task(service.execute(session, transition)) + try: + await asyncio.wait_for(reached.wait(), timeout=1) + task.cancel("caller cancelled") + await asyncio.wait_for(cancel_seen.wait(), timeout=1) + if repeat: + await asyncio.wait_for(close_entered.wait(), timeout=1) + task.cancel("cancelled again") + await asyncio.sleep(0) + self.assertFalse(task.done()) + release_close.set() + with self.assertRaises(asyncio.CancelledError) as raised: + await asyncio.wait_for(task, timeout=1) + self.assertEqual(raised.exception.args, ("caller cancelled",)) + finally: + release_close.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + self.assert_recorder_closed(service) + recorder = self.recorders[-1] + if stage in {"talk", "cleanup"}: + self.assertIsNotNone(recorder.finished_path) + if recorder.finished_path is not None: + self.assertFalse(recorder.finished_path.exists()) + if stage == "cleanup": + self.assertEqual(session.state, CallState.REMOTE_ENDED) + else: + self.assertEqual(session.end_reason, CallEndReason.SHUTDOWN) + self.assertEqual(session.error_code, "cancelled") + self.assertTrue(fixtures.FakeVoice.instances[-1].closed) + if stage != "start": + client = fixtures.FakeClient.instances[-1] + self.assertTrue(client.calls.closed) + self.assertEqual(client.actions.count("hangup"), int(stage != "open")) + voice = fixtures.FakeVoice.instances[-1] + self.assertTrue(all(track.stopped for track in getattr(voice, "tracks", []))) + self.assertTrue(all(sink.closed for sink in getattr(voice, "sinks", []))) + + async def test_cancellation_finishes_recording_at_every_stage(self): + for stage in ("start", "open", "dial", "ring", "bridge", "talk", "cleanup"): + with self.subTest(stage=stage): + await self.cancel_at_stage(stage) + + async def test_repeated_cancel_waits_for_cleanup(self): + await self.cancel_at_stage("talk", repeat=True) + + async def test_hangup_error_does_not_replace_cancellation(self): + await self.cancel_at_stage("talk", hangup_fails=True) + + async def test_normal_success_still_returns_recording(self): + greet = fixtures.FakeVoice.greet_once + + async def record_audio(voice): + await greet(voice) + recorder = voice.options["recorder"] + recorder.set_generation(1) + recorder.add_bot(b"\x01\x00" * 480, 1) + + for remote_hangup in (True, False): + with self.subTest(remote_hangup=remote_hangup): + fixtures.FakeClient.remote_hangup = remote_hangup + session, transition = self.make_session() + service = self.make_service(talk=0.02) + with patch.object(fixtures.FakeVoice, "greet_once", record_audio): + result = await service.execute(session, transition) + self.assert_recorder_closed(service) + self.assertEqual(session.state, CallState.REMOTE_ENDED if remote_hangup else CallState.TALK_TIMEOUT) + self.assertIsNotNone(result.recording_path) + try: + self.assertTrue(result.recording_path.exists()) + finally: + result.recording_path.unlink(missing_ok=True) + + async def test_close_failures_preserve_cleanup_order(self): + for failing_resource in ("calls", "voice"): + with self.subTest(failing_resource=failing_resource): + session, transition = self.make_session() + service = self.make_service() + actions = [] + calls_close = fixtures.FakeCalls.closeAsync + voice_close = fixtures.FakeVoice.close + + async def close_calls(calls): + actions.append("calls") + await calls_close(calls) + if failing_resource == "calls": + raise RuntimeError("RTC close failed") + + async def close_voice(voice): + actions.append("voice") + await voice_close(voice) + if failing_resource == "voice": + raise RuntimeError("voice close failed") + + finish = CallRecorder.finish + + def finish_recording(recorder): + actions.append("recording") + return finish(recorder) + + with ( + patch.object(fixtures.FakeCalls, "closeAsync", close_calls), + patch.object(fixtures.FakeVoice, "close", close_voice), + patch.object(CallRecorder, "finish", finish_recording), + ): + await service.execute(session, transition) + self.assertEqual(actions, ["calls", "voice", "recording"]) + self.assertEqual(session.state, CallState.REMOTE_ENDED) + self.assert_recorder_closed(service) + + async def test_failed_error_transition_still_finishes_recorder(self): + fixtures.FakeClient.open_error = True + session, apply = self.make_session() + service = self.make_service() + + def transition(call_session, event, **options): + if event == CallEvent.INTERNAL_ERROR: + raise ValueError("error callback failed") + return apply(call_session, event, **options) + + with self.assertRaisesRegex(ValueError, "error callback failed"): + await service.execute(session, transition) + self.assert_recorder_closed(service) + self.assertTrue(fixtures.FakeClient.instances[-1].calls.closed) + self.assertTrue(fixtures.FakeVoice.instances[-1].closed) + + async def test_failed_cancellation_transition_preserves_original_cancellation(self): + session, apply = self.make_session() + service = self.make_service() + cancellation = asyncio.CancelledError("original cancellation") + + def transition(call_session, event, **options): + if event == CallEvent.SHUTDOWN_REQUESTED: + raise ValueError("shutdown callback failed") + return apply(call_session, event, **options) + + with patch.object(fixtures.FakeVoice, "start", side_effect=cancellation): + with self.assertRaises(asyncio.CancelledError) as raised: + await service.execute(session, transition) + self.assertIs(raised.exception, cancellation) + self.assert_recorder_closed(service) + self.assertTrue(fixtures.FakeVoice.instances[-1].closed) + + async def test_sink_creation_failure_stops_output_track(self): + session, transition = self.make_session() + service = self.make_service() + with patch.object(fixtures.FakeVoice, "new_input_sink", side_effect=RuntimeError("sink failed")): + await service.execute(session, transition) + voice = fixtures.FakeVoice.instances[-1] + self.assertTrue(voice.tracks[0].stopped) + self.assertEqual(session.state, CallState.FAILED) + self.assertEqual(session.error_code, "RuntimeError") + self.assert_recorder_closed(service) + + async def test_sink_close_failure_stops_track_and_closes_voice(self): + session, transition = self.make_session() + service = self.make_service() + with patch.object(fixtures.FakeSink, "close", side_effect=RuntimeError("sink close failed")): + await service.execute(session, transition) + voice = fixtures.FakeVoice.instances[-1] + self.assertTrue(voice.tracks[0].stopped) + self.assertTrue(voice.closed) + self.assertEqual(session.state, CallState.REMOTE_ENDED) + self.assert_recorder_closed(service) diff --git a/whatsapp_chatbot_python/calls/contracts.py b/whatsapp_chatbot_python/calls/contracts.py index 730f992..fded04d 100644 --- a/whatsapp_chatbot_python/calls/contracts.py +++ b/whatsapp_chatbot_python/calls/contracts.py @@ -19,7 +19,7 @@ def __call__( class CallExecutor(Protocol): - """Execute calls serially; request_stop permanently stops this executor.""" + """request_stop permanently stops this executor.""" async def execute( self, session: CallSession, transition: TransitionCallback diff --git a/whatsapp_chatbot_python/calls/service.py b/whatsapp_chatbot_python/calls/service.py index c52f143..6a30c60 100644 --- a/whatsapp_chatbot_python/calls/service.py +++ b/whatsapp_chatbot_python/calls/service.py @@ -1,7 +1,10 @@ from __future__ import annotations from contextlib import suppress +from dataclasses import dataclass +from functools import partial +from pathlib import Path from whatsapp_api_client_python.API import GreenAPI -from whatsapp_api_client_python.tools.voip import CallAudio +from whatsapp_api_client_python.tools.voip import CallAudio, CallsConnection from .contracts import TransitionCallback from .realtime_voice import VoiceBotSession from .recording import CallRecorder @@ -17,6 +20,26 @@ INITIAL_STATE_TIMEOUT_SECONDS = 10 +@dataclass(slots=True) +class _CallExecution: + """Resources and progress of one execution, including partially completed setup.""" + + session: CallSession + transition: TransitionCallback + recorder: CallRecorder + runtime: CallRuntime | None = None + voice: VoiceBotSession | None = None + api: GreenAPI | None = None + calls: CallsConnection | None = None + initial_state: asyncio.Future[str] | None = None + bridge_task: asyncio.Task[None] | None = None + dial_started: bool = False + dialed: bool = False + execution_completed: bool = False + cancellation: asyncio.CancelledError | None = None + recording_path: Path | None = None + + class WhatsAppCallService: def __init__( self, @@ -56,187 +79,273 @@ def request_stop(self) -> None: async def execute( self, - call_session: CallSession, + session: CallSession, transition: TransitionCallback, ) -> CallExecutionResult: if self._stop_requested: - transition(call_session, CallEvent.SHUTDOWN_REQUESTED, error_code="shutdown") + transition(session, CallEvent.SHUTDOWN_REQUESTED, error_code="shutdown") return CallExecutionResult() - recorder = CallRecorder(self._logger) + execution = _CallExecution(session, transition, CallRecorder(self._logger)) + + try: + await self._execute_call(execution) + except asyncio.CancelledError as error: + self._record_cancellation(execution, error) + raise + finally: + await self._finalize_call(execution) + + return CallExecutionResult(execution.recording_path) + + async def _execute_call(self, execution: _CallExecution) -> None: + # Setup failures propagate to the caller; call failures become FSM events. + await self._prepare_call(execution) + + try: + await self._run_call(execution) + except Exception as error: + await self._handle_call_failure(execution, error) + execution.execution_completed = True + + async def _prepare_call(self, execution: _CallExecution) -> None: runtime = CallRuntime( ring_timeout_seconds=self._ring_timeout, talk_timeout_seconds=self._talk_timeout, bridge_timeout_seconds=BRIDGE_NEGOTIATION_TIMEOUT_SECONDS, ) + execution.runtime = runtime self._active_loop = asyncio.get_running_loop() self._active_runtime = runtime if self._stop_requested: runtime.events.put_nowait(RuntimeEvent.shutdown()) - voice = VoiceBotSession( + execution.voice = VoiceBotSession( api_key=self._openai_api_key, model=self._realtime_model, voice=self._realtime_voice, - language=call_session.language, + language=execution.session.language, on_error=lambda error: runtime.events.put_nowait(RuntimeEvent.voice_failed(error)), - recorder=recorder, + recorder=execution.recorder, ) - async def make_audio_session() -> CallAudio: - track = await voice.new_output_track() + await execution.voice.start() - try: - sink = voice.new_input_sink() - except BaseException: - track.stop() - raise + execution.api = GreenAPI(self._id_instance, self._api_token_instance, host=self._api_url) - async def close_audio() -> None: - try: - await sink.close() - finally: - track.stop() + execution.calls = execution.api.voip.connect( + audio_factory=partial(self._create_audio_session, execution.voice), + ) - return CallAudio(track, sink.attach, close_audio) + execution.initial_state = asyncio.get_running_loop().create_future() - try: - await voice.start() - api = GreenAPI(self._id_instance, self._api_token_instance, host=self._api_url) - calls = api.voip.connect(audio_factory=make_audio_session) - except Exception: - await voice.close() - recorder.finish() - self._active_runtime = None - self._active_loop = None + self._register_listeners(execution) + async def _create_audio_session(self, voice: VoiceBotSession) -> CallAudio: + track = await voice.new_output_track() + + try: + sink = voice.new_input_sink() + except BaseException: + track.stop() raise - bridge_task: asyncio.Task[None] | None = None - dialed = False - initial_state: asyncio.Future[str] = asyncio.get_running_loop().create_future() - dial_started = False + return CallAudio(track, sink.attach, partial(self._close_audio_session, sink, track)) - def on_state(state) -> None: - if not initial_state.done(): - initial_state.set_result(state.state) + @staticmethod + async def _close_audio_session(sink, track) -> None: + try: + await sink.close() + finally: + track.stop() - if dial_started: - runtime.events.put_nowait(RuntimeEvent.server_state(state.state, state.reason)) + def _register_listeners(self, execution: _CallExecution) -> None: + calls = execution.calls + runtime = execution.runtime - def on_end_call(detail) -> None: - # NOTE: If the server returned the reason, it is a handled case. - # Otherwise it falls into the error handler. - runtime.events.put_nowait(RuntimeEvent.end_call(detail.get("cause", None))) + calls.on("state", partial(self._on_state, execution)) - def on_disconnect(detail) -> None: - permanent = bool(detail.get("permanent")) + calls.on("end_call", lambda detail: runtime.events.put_nowait( + RuntimeEvent.end_call(detail.get("cause", None)), + )) - if permanent and not initial_state.done(): - initial_state.set_exception(RuntimeError("callsRtc connection refused")) + calls.on("disconnect", partial(self._on_disconnect, execution)) - runtime.events.put_nowait(RuntimeEvent.disconnect(permanent)) + calls.on("error", lambda detail: runtime.events.put_nowait( + RuntimeEvent.bridge_failed(RuntimeError("callsRtc reported an error")), + )) - def on_bridge_done(task: asyncio.Task[None]) -> None: - if task.cancelled(): - return + @staticmethod + def _on_state(execution: _CallExecution, state) -> None: + if not execution.initial_state.done(): + execution.initial_state.set_result(state.state) - error = task.exception() - result = RuntimeEvent.bridge_failed(error) if error else RuntimeEvent.bridge_ready() + if execution.dial_started: + execution.runtime.events.put_nowait(RuntimeEvent.server_state(state.state, state.reason)) - runtime.events.put_nowait(result) + @staticmethod + def _on_disconnect(execution: _CallExecution, detail) -> None: + permanent = bool(detail.get("permanent")) - calls.on("state", on_state) - calls.on("end_call", on_end_call) - calls.on("disconnect", on_disconnect) + if permanent and not execution.initial_state.done(): + execution.initial_state.set_exception(RuntimeError("callsRtc connection refused")) - calls.on( - "error", - lambda detail: runtime.events.put_nowait( - RuntimeEvent.bridge_failed(RuntimeError("callsRtc reported an error")) - ), - ) + execution.runtime.events.put_nowait(RuntimeEvent.disconnect(permanent)) - try: - await calls.openAsync(timeout=INITIAL_STATE_TIMEOUT_SECONDS) + @staticmethod + def _on_bridge_done(execution: _CallExecution, task: asyncio.Task[None]) -> None: + if task.cancelled(): + return + + error = task.exception() + event = RuntimeEvent.bridge_failed(error) if error else RuntimeEvent.bridge_ready() + + execution.runtime.events.put_nowait(event) - state = await asyncio.wait_for(initial_state, timeout=INITIAL_STATE_TIMEOUT_SECONDS) + async def _run_call(self, execution: _CallExecution) -> None: + await execution.calls.openAsync(timeout=INITIAL_STATE_TIMEOUT_SECONDS) - if state != "idle": - raise RuntimeError(f"Instance already has a call: {state}") + state = await asyncio.wait_for(execution.initial_state, timeout=INITIAL_STATE_TIMEOUT_SECONDS) - if self._stop_requested: - transition(call_session, CallEvent.SHUTDOWN_REQUESTED, error_code="shutdown") + if state != "idle": + raise RuntimeError(f"Instance already has a call: {state}") + + if self._stop_requested: + execution.transition(execution.session, CallEvent.SHUTDOWN_REQUESTED, error_code="shutdown") + else: + execution.dial_started = True + + await execution.api.voip.dialAsync(execution.session.chat_id) + + execution.dialed = True + + execution.transition(execution.session, CallEvent.DIAL_ACCEPTED) + execution.runtime.start_ringing() + + # Bridge completion confirms SDP, not ICE/DTLS or live audio. + execution.bridge_task = asyncio.create_task(execution.calls.startAudioAsync()) + + execution.bridge_task.add_done_callback(partial(self._on_bridge_done, execution)) + + await self._process_events(execution) + + async def _process_events(self, execution: _CallExecution) -> None: + while execution.session.state not in TERMINAL_STATES: + event = await execution.runtime.next_event(execution.session.state) + + if event is None: + await self._handle_timeout( + execution.runtime, execution.api, execution.session, execution.transition, + ) else: - dial_started = True + await self._handle_event( + event, execution.runtime, execution.api, execution.voice, + execution.recorder, execution.session, execution.transition, + ) + + async def _handle_call_failure(self, execution: _CallExecution, error: Exception) -> None: + self._logger.error( + "VoIP execution failed: session=%s error=%s", + execution.session.session_id, + type(error).__name__, + ) - await api.voip.dialAsync(call_session.chat_id) + execution.transition( + execution.session, CallEvent.INTERNAL_ERROR, + error_code=type(error).__name__, + ) - dialed = True + if execution.dialed: + await self._safe_hang_up(execution.api) - transition(call_session, CallEvent.DIAL_ACCEPTED) - runtime.start_ringing() + @staticmethod + def _record_cancellation(execution: _CallExecution, error: asyncio.CancelledError) -> None: + if execution.cancellation is None: + execution.cancellation = error - # The library owns signaling and WebRTC; the voice session supplies audio. - # Bridge completion confirms the SDP answer, not ICE/DTLS or live audio. - bridge_task = asyncio.create_task(calls.startAudioAsync()) - bridge_task.add_done_callback(on_bridge_done) + # A failing callback must not prevent resource cleanup. + with suppress(Exception): + execution.transition( + execution.session, CallEvent.SHUTDOWN_REQUESTED, error_code="cancelled", + ) - while call_session.state not in TERMINAL_STATES: - event = await runtime.next_event(call_session.state) + async def _wait_for_cleanup(self, coroutine, execution: _CallExecution) -> None: + # Own the cleanup task until completion, including repeated cancel(). + task = asyncio.create_task(coroutine) - if event is None: - await self._handle_timeout(runtime, api, call_session, transition) - else: - await self._handle_event(event, runtime, api, voice, recorder, call_session, transition) - except Exception as error: - self._logger.error( - "VoIP execution failed: session=%s error=%s", - call_session.session_id, - type(error).__name__, - ) + while True: + try: + await asyncio.shield(task) + return + except asyncio.CancelledError as error: + self._record_cancellation(execution, error) - transition( - call_session, CallEvent.INTERNAL_ERROR, - error_code=type(error).__name__, - ) + if task.cancelled(): + raise - if dialed: - await self._safe_hang_up(api) + async def _finalize_call(self, execution: _CallExecution) -> None: + try: + await self._close_call(execution) finally: - with suppress(Exception): - await asyncio.wait_for(calls.closeAsync(), timeout=self._shutdown_timeout) + self._finish_recording(execution) - with suppress(Exception): - await asyncio.wait_for(voice.close(), timeout=self._shutdown_timeout) + if execution.cancellation is not None: + raise execution.cancellation - if bridge_task is not None: - if not bridge_task.done(): - bridge_task.cancel() + async def _close_call(self, execution: _CallExecution) -> None: + try: + await self._wait_for_cleanup(self._close_connections(execution), execution) + finally: + await self._hang_up_after_cancellation(execution) - with suppress(Exception): - await asyncio.wait_for( - asyncio.gather(bridge_task, return_exceptions=True), - timeout=self._shutdown_timeout, - ) + async def _close_connections(self, execution: _CallExecution) -> None: + # Preserve shutdown order: RTC, voice session, then the bridge task. + if execution.calls is not None: + with suppress(Exception): + await asyncio.wait_for(execution.calls.closeAsync(), timeout=self._shutdown_timeout) - self._active_runtime = None - self._active_loop = None + if execution.voice is not None: + with suppress(Exception): + await asyncio.wait_for(execution.voice.close(), timeout=self._shutdown_timeout) - # Encoding runs after the RTC and Realtime connections have closed. - path = recorder.finish() + if execution.bridge_task is not None: + if not execution.bridge_task.done(): + execution.bridge_task.cancel() - if call_session.state not in {CallState.REMOTE_ENDED, CallState.TALK_TIMEOUT}: - # Preserve the established policy: discard partial failed calls. - if path is not None: - path.unlink(missing_ok=True) + with suppress(Exception): + await asyncio.wait_for( + asyncio.gather(execution.bridge_task, return_exceptions=True), + timeout=self._shutdown_timeout, + ) + + async def _hang_up_after_cancellation(self, execution: _CallExecution) -> None: + # The dial request may have reached the server before its response arrived. + if execution.cancellation is not None and execution.dial_started: + with suppress(Exception): + await self._wait_for_cleanup( + asyncio.wait_for(self._safe_hang_up(execution.api), timeout=self._shutdown_timeout), + execution, + ) + + def _finish_recording(self, execution: _CallExecution) -> None: + self._active_runtime = None + self._active_loop = None + path = execution.recorder.finish() + + keep_recording = ( + execution.execution_completed + and execution.cancellation is None + and execution.session.state in {CallState.REMOTE_ENDED, CallState.TALK_TIMEOUT} + ) + if path is not None and not keep_recording: + path.unlink(missing_ok=True) path = None - return CallExecutionResult(path) + execution.recording_path = path async def _handle_event( self, From c51d2db88b826034cc391982fdaf9e99fef5ef85 Mon Sep 17 00:00:00 2001 From: kennyromanov Date: Wed, 7 Oct 2026 17:31:47 +0500 Subject: [PATCH 4/4] fix: The issue with the cleanup has been fixed x2; The python bug has been worked-around --- tests/calls/test_call_runtime.py | 38 ++++++++++++++++++++++++ whatsapp_chatbot_python/calls/runtime.py | 5 +++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/calls/test_call_runtime.py b/tests/calls/test_call_runtime.py index 2ea739d..d95baec 100644 --- a/tests/calls/test_call_runtime.py +++ b/tests/calls/test_call_runtime.py @@ -1,5 +1,6 @@ from whatsapp_chatbot_python.calls.models import CallState from whatsapp_chatbot_python.calls.runtime import CallRuntime, RuntimeEvent, RuntimeEventType +import asyncio import unittest @@ -19,5 +20,42 @@ async def test_queued_event_wins_over_expired_timer(self) -> None: self.assertEqual(event.type, RuntimeEventType.BRIDGE_READY) + async def test_empty_queue_returns_none_when_timer_expires(self) -> None: + runtime = CallRuntime(0, 1, 1) + runtime.start_ringing() + self.assertIsNone(await runtime.next_event(CallState.RINGING)) + + async def test_cancellation_is_not_swallowed_when_event_becomes_ready(self) -> None: + waiting = asyncio.Event() + + class ObservedQueue(asyncio.Queue): + async def get(self): + waiting.set() + return await super().get() + + runtime = CallRuntime(5, 1, 1, events=ObservedQueue()) + runtime.start_ringing() + + async def consume(): + event = await runtime.next_event(CallState.RINGING) + # Allow cancellation to arrive even if the event was delivered first. + await asyncio.sleep(0) + return event + + task = asyncio.create_task(consume()) + try: + await asyncio.wait_for(waiting.wait(), timeout=1) + runtime.events.put_nowait(RuntimeEvent.bridge_ready()) + # Resume the queue reader, then cancel before wait_for's parent resumes. + asyncio.get_running_loop().call_soon(task.cancel, "caller cancelled") + with self.assertRaises(asyncio.CancelledError) as raised: + await asyncio.wait_for(task, timeout=1) + self.assertEqual(raised.exception.args, ("caller cancelled",)) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + if __name__ == "__main__": unittest.main() diff --git a/whatsapp_chatbot_python/calls/runtime.py b/whatsapp_chatbot_python/calls/runtime.py index 8417b97..508daeb 100644 --- a/whatsapp_chatbot_python/calls/runtime.py +++ b/whatsapp_chatbot_python/calls/runtime.py @@ -91,7 +91,10 @@ async def next_event(self, state: CallState) -> RuntimeEvent | None: timeout = self._next_timeout(state) try: - return await asyncio.wait_for(self.events.get(), timeout=timeout) + # Python 3.11 wait_for() can swallow cancellation when an event becomes ready. Wait in the current task + # instead: https://github.com/python/cpython/issues/86296 + async with asyncio.timeout(timeout): + return await self.events.get() except TimeoutError: return None