diff --git a/README.md b/README.md index c41ac45..34e607d 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,36 @@ async with DecartClient(api_key=os.getenv("DECART_API_KEY")) as client: f.write(data) ``` +### Realtime fast mode + +Realtime sessions accept an optional `speed` on `RealtimeConnectOptions`, alongside `resolution`. +Fast mode (`speed="fast"`) serves the session from a higher-compute tier for lower latency and +higher throughput; output quality is unchanged. It is currently available for `lucy-2.5` / +`lucy-latest` and `lucy-vton-3.5` / `lucy-vton-latest`, in the US region only, and is billed at +2x the standard realtime rate for those models. Other models ignore the option (the SDK emits a +warning). Omit it (the default) for standard mode. + +```python +from decart import DecartClient, models +from decart.realtime import RealtimeClient, RealtimeConnectOptions + +client = DecartClient(api_key=os.getenv("DECART_API_KEY")) +realtime = await RealtimeClient.connect( + base_url=client.realtime_base_url, + api_key=client.api_key, + local_track=local_track, + options=RealtimeConnectOptions( + model=models.realtime("lucy-2.5"), + on_remote_stream=on_remote_stream, + speed="fast", # omit for standard mode + ), +) +``` + +Each model definition lists the speed tiers it advertises via `ModelDefinition.supported_speeds` +(for example `models.realtime("lucy-2.5").supported_speeds == ("fast",)`). See the +[realtime docs](https://docs.platform.decart.ai/sdks/python) for the full realtime API. + ## Development ### Setup with UV diff --git a/decart/__init__.py b/decart/__init__.py index b7ce1d0..a6c782e 100644 --- a/decart/__init__.py +++ b/decart/__init__.py @@ -12,7 +12,13 @@ QueueResultError, TokenCreateError, ) -from .models import models, ModelDefinition, CustomModelDefinition, VideoRestyleInput +from .models import ( + models, + ModelDefinition, + CustomModelDefinition, + VideoRestyleInput, + RealtimeSpeed, +) from .types import FileInput, ModelState, Prompt from .queue import ( QueueClient, @@ -71,6 +77,7 @@ "ModelDefinition", "CustomModelDefinition", "VideoRestyleInput", + "RealtimeSpeed", "FileInput", "ModelState", "Prompt", diff --git a/decart/models.py b/decart/models.py index 5a3ede9..9ce5317 100644 --- a/decart/models.py +++ b/decart/models.py @@ -1,5 +1,5 @@ import warnings -from typing import Literal, Optional, Generic, TypeVar +from typing import Any, Literal, Optional, Generic, TypeVar from pydantic import BaseModel, Field, ConfigDict, model_validator from .errors import ModelNotFoundError from .types import FileInput @@ -41,6 +41,10 @@ ] Model = Literal[RealTimeModels, VideoModels, ImageModels] +RealtimeSpeed = Literal["fast"] +"""Realtime speed tier. ``"fast"`` serves the session from a higher-compute tier +for lower latency and higher throughput; omit it (the default) for standard mode.""" + MODEL_ALIASES: dict[str, str] = { # Video aliases "lucy-pro-v2v": "lucy-clip", @@ -64,6 +68,29 @@ def _warn_deprecated(model: str) -> None: ) +_warned_unsupported_speeds: set[tuple[str, str]] = set() + + +def _warn_unsupported_speed(model: "ModelDefinition[Any]", speed: str) -> None: + """Warn once per (model, speed) when a realtime speed tier the model does not + advertise is requested. The option is still sent; the server ignores it for + models without the tier.""" + if speed in model.supported_speeds: + return + key = (model.name, speed) + if key in _warned_unsupported_speeds: + return + _warned_unsupported_speeds.add(key) + warnings.warn( + f'Model "{model.name}" does not support speed="{speed}"; the option is ignored by the ' + "server for this model. Fast mode is currently available for lucy-2.5 / lucy-latest and " + "lucy-vton-3.5 / lucy-vton-latest only. See https://docs.platform.decart.ai/models " + "for details.", + UserWarning, + stacklevel=3, + ) + + # Type variable for model name ModelT = TypeVar("ModelT", bound=str) @@ -79,6 +106,8 @@ class ModelDefinition(DecartBaseModel, Generic[ModelT]): width: int = Field(ge=1) height: int = Field(ge=1) input_schema: Optional[type[BaseModel]] = None + supported_speeds: tuple[RealtimeSpeed, ...] = () + """Realtime speed tiers this model advertises (empty for standard mode only).""" # Type aliases for model definitions that support specific APIs @@ -182,6 +211,7 @@ class ImageToImageInput(DecartBaseModel): fps=30, width=1280, height=720, + supported_speeds=("fast",), ), "lucy-restyle-2": ModelDefinition( name="lucy-restyle-2", @@ -197,6 +227,7 @@ class ImageToImageInput(DecartBaseModel): fps=30, width=1088, height=624, + supported_speeds=("fast",), ), # Server-side alias currently resolves to lucy-vton-3.5. "lucy-vton-latest": ModelDefinition( @@ -205,6 +236,7 @@ class ImageToImageInput(DecartBaseModel): fps=30, width=1280, height=720, + supported_speeds=("fast",), ), "lucy-vton-3.5": ModelDefinition( name="lucy-vton-3.5", @@ -212,6 +244,7 @@ class ImageToImageInput(DecartBaseModel): fps=30, width=1280, height=720, + supported_speeds=("fast",), ), "lucy-restyle-latest": ModelDefinition( name="lucy-restyle-latest", diff --git a/decart/realtime/__init__.py b/decart/realtime/__init__.py index bc3e3e4..5a2c7c6 100644 --- a/decart/realtime/__init__.py +++ b/decart/realtime/__init__.py @@ -6,7 +6,7 @@ decode_subscribe_token, ) from .messages import GenerationTickMessage -from .types import RealtimeConnectOptions, ConnectionState +from .types import RealtimeConnectOptions, ConnectionState, RealtimeSpeed __all__ = [ "RealtimeClient", @@ -18,4 +18,5 @@ "GenerationTickMessage", "RealtimeConnectOptions", "ConnectionState", + "RealtimeSpeed", ] diff --git a/decart/realtime/client.py b/decart/realtime/client.py index d646bcd..ce4fef8 100644 --- a/decart/realtime/client.py +++ b/decart/realtime/client.py @@ -19,6 +19,7 @@ from .types import ConnectionState, RealtimeConnectOptions from ..types import FileInput from ..errors import DecartSDKError, InvalidInputError, WebRTCError +from ..models import _warn_unsupported_speed from ..process.request import file_input_to_bytes if TYPE_CHECKING: @@ -174,6 +175,9 @@ async def connect( ) if options.resolution is not None: ws_url += f"&resolution={quote(options.resolution)}" + if options.speed is not None: + _warn_unsupported_speed(options.model, options.speed) + ws_url += f"&speed={quote(options.speed)}" config = LiveKitConfiguration( livekit_url=ws_url, diff --git a/decart/realtime/types.py b/decart/realtime/types.py index 2345dd8..d4c074b 100644 --- a/decart/realtime/types.py +++ b/decart/realtime/types.py @@ -1,6 +1,6 @@ from typing import Literal, Callable, Optional, TYPE_CHECKING from dataclasses import dataclass -from ..models import ModelDefinition +from ..models import ModelDefinition, RealtimeSpeed from ..types import ModelState if TYPE_CHECKING: @@ -10,6 +10,8 @@ ConnectionState = Literal["connecting", "connected", "generating", "disconnected", "reconnecting"] VideoCodec = Literal["h264", "vp9"] +__all__ = ["ConnectionState", "VideoCodec", "RealtimeSpeed", "RealtimeConnectOptions"] + @dataclass class RealtimeConnectOptions: @@ -18,3 +20,9 @@ class RealtimeConnectOptions: initial_state: Optional[ModelState] = None resolution: Optional[Literal["720p", "1080p"]] = None preferred_video_codec: VideoCodec = "h264" + speed: Optional[RealtimeSpeed] = None + """Realtime speed tier. ``"fast"`` serves the session from a higher-compute tier for + lower latency and higher throughput; output quality is unchanged. Currently available + for lucy-2.5 / lucy-latest and lucy-vton-3.5 / lucy-vton-latest, in the US region only, + and billed at 2x the standard realtime rate for those models. Other models ignore the + option. Omit it (the default) for standard mode.""" diff --git a/examples/README.md b/examples/README.md index 45721eb..e93289c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -35,6 +35,13 @@ pip install decart[realtime] - **`realtime_synthetic.py`** - Publish synthetic colored frames through LiveKit - **`realtime_file.py`** - Publish frames from a video file through LiveKit +Fast mode: pass `speed="fast"` on `RealtimeConnectOptions` to serve the session from a +higher-compute tier for lower latency and higher throughput (output quality is unchanged). It is +currently available for `lucy-2.5` / `lucy-latest` and `lucy-vton-3.5` / `lucy-vton-latest`, in +the US region only, and is billed at 2x the standard realtime rate for those models. Other models +ignore the option. Omit it (the default) for standard mode. The playground exposes it as +`python playground/playground.py --model lucy-2.5 --speed fast`. + ### Running Examples `process_image.py` uses the bundled `examples/files/image.png` asset. diff --git a/examples/realtime_synthetic.py b/examples/realtime_synthetic.py index d06ce37..f89ff16 100644 --- a/examples/realtime_synthetic.py +++ b/examples/realtime_synthetic.py @@ -127,6 +127,10 @@ def on_error(error): ), image=Path("examples/files/image.png"), ), + # Fast mode: uncomment with a supporting model (lucy-2.5 / lucy-latest, + # lucy-vton-3.5 / lucy-vton-latest). US region only, billed at 2x the + # standard realtime rate; other models ignore it. Omit for standard mode. + # speed="fast", ), ) diff --git a/playground/playground.py b/playground/playground.py index 9d22364..e17ebc5 100644 --- a/playground/playground.py +++ b/playground/playground.py @@ -49,7 +49,7 @@ def _check_deps() -> None: from livekit import rtc # noqa: E402 from decart import DecartClient, models # noqa: E402 -from decart.models import RealTimeModels # noqa: E402 +from decart.models import RealtimeSpeed, RealTimeModels # noqa: E402 from decart.realtime.client import RealtimeClient # noqa: E402 from decart.realtime.types import RealtimeConnectOptions # noqa: E402 from decart.types import ModelState, Prompt # noqa: E402 @@ -119,6 +119,15 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--api-key", "-k", help="API key (or set DECART_API_KEY)") parser.add_argument("--image", "-i", help="Optional reference image") parser.add_argument("--prompt", "-p", help="Initial prompt text") + parser.add_argument( + "--speed", + choices=["fast"], + help=( + "Realtime speed tier. 'fast' uses a higher-compute tier for lower latency " + "(lucy-2.5 / lucy-vton-3.5 and their -latest aliases only, US region, billed at 2x). " + "Omit for standard mode." + ), + ) parser.add_argument("--camera", "-c", type=int, default=0, help="Camera device index") parser.add_argument("--no-local", action="store_true", help="Hide local camera feed") parser.add_argument("--verbose", "-v", action="store_true", help="Enable debug logging") @@ -156,8 +165,10 @@ async def run() -> None: model_name = args.model or select_model_interactive() model = models.realtime(cast(RealTimeModels, model_name)) + speed = cast(Optional[RealtimeSpeed], args.speed) print(f"\n Model : {model_name}") print(f" Res : {model.width}x{model.height} @ {model.fps}fps") + print(f" Speed : {speed or 'standard'}") if args.image and not Path(args.image).exists(): print(f"\nError: Image not found: {args.image}") @@ -223,6 +234,7 @@ def _read_prompts() -> None: model=model, on_remote_stream=on_remote_stream, initial_state=initial_state, + speed=speed, ), ) realtime.on("connection_change", on_connection_change) diff --git a/tests/test_models.py b/tests/test_models.py index 73b4c3f..3e1ada2 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,7 +1,7 @@ import warnings import pytest -from decart import models, DecartSDKError, ModelDefinition -from decart.models import _warned_aliases +from decart import models, DecartSDKError, ModelDefinition, RealtimeSpeed +from decart.models import _MODELS, _warned_aliases def test_canonical_realtime_models() -> None: @@ -183,6 +183,35 @@ def test_latest_aliases_no_deprecation_warning() -> None: assert len(w) == 0 +FAST_REALTIME_MODELS = {"lucy-2.5", "lucy-latest", "lucy-vton-3.5", "lucy-vton-latest"} + + +def test_realtime_fast_speed_capability_pinned_to_lucy_2_5_and_vton_3_5() -> None: + # Matches the JS SDK registry: only these realtime entries advertise speed="fast". + fast_models = { + name for name, model in _MODELS["realtime"].items() if "fast" in model.supported_speeds + } + assert fast_models == FAST_REALTIME_MODELS + + for name, model in _MODELS["realtime"].items(): + if name in FAST_REALTIME_MODELS: + assert model.supported_speeds == ("fast",) + else: + assert model.supported_speeds == () + + +def test_non_realtime_models_advertise_no_speeds() -> None: + for surface in ("video", "image"): + for model in _MODELS[surface].values(): + assert model.supported_speeds == () + + +def test_realtime_speed_literal_is_fast_only() -> None: + from typing import get_args + + assert get_args(RealtimeSpeed) == ("fast",) + + def test_custom_model_definition_allows_arbitrary_model_names() -> None: model = ModelDefinition( name="lucy_2_rt_preview", @@ -194,6 +223,7 @@ def test_custom_model_definition_allows_arbitrary_model_names() -> None: assert model.name == "lucy_2_rt_preview" assert model.input_schema is None + assert model.supported_speeds == () def test_invalid_model() -> None: diff --git a/tests/test_realtime_unit.py b/tests/test_realtime_unit.py index bfe98de..a5f3c47 100644 --- a/tests/test_realtime_unit.py +++ b/tests/test_realtime_unit.py @@ -138,7 +138,7 @@ async def test_realtime_connect_accepts_custom_model_definition(): await realtime_client.disconnect() -async def _connect_and_capture_url(resolution=None) -> str: +async def _connect_and_capture_url(resolution=None, speed=None, model="lucy-2.1") -> str: from decart.realtime.types import RealtimeConnectOptions client = DecartClient(api_key="test-key") @@ -155,13 +155,17 @@ async def _connect_and_capture_url(resolution=None) -> str: mock_session.close = AsyncMock() mock_session_cls.return_value = mock_session - kwargs = {"resolution": resolution} if resolution is not None else {} + kwargs = {} + if resolution is not None: + kwargs["resolution"] = resolution + if speed is not None: + kwargs["speed"] = speed realtime_client = await RealtimeClient.connect( base_url=client.realtime_base_url, api_key=client.api_key, local_track=MagicMock(), options=RealtimeConnectOptions( - model=models.realtime("lucy-2.1"), + model=models.realtime(model), on_remote_stream=lambda t: None, **kwargs, ), @@ -185,6 +189,121 @@ async def test_realtime_connect_appends_resolution_720p(): assert "&resolution=720p" in url +@pytest.mark.asyncio +async def test_realtime_connect_omits_speed_when_unset(): + url = await _connect_and_capture_url(model="lucy-2.5") + assert "speed" not in url + assert url == ( + "wss://api3.decart.ai/v1/stream" + "?api_key=test-key&model=lucy-2.5&livekit_early_room_info=true" + ) + + +@pytest.mark.asyncio +async def test_realtime_connect_appends_speed_fast(): + url = await _connect_and_capture_url(speed="fast", model="lucy-2.5") + assert url == ( + "wss://api3.decart.ai/v1/stream" + "?api_key=test-key&model=lucy-2.5&livekit_early_room_info=true&speed=fast" + ) + assert url.count("speed=") == 1 + + +@pytest.mark.asyncio +async def test_realtime_connect_appends_speed_after_resolution(): + url = await _connect_and_capture_url(resolution="720p", speed="fast", model="lucy-2.5") + assert url.endswith("&livekit_early_room_info=true&resolution=720p&speed=fast") + + +def _speed_warnings(recorded) -> list: + # Unrelated warnings (e.g. aiohttp "Unclosed client session" ResourceWarnings raised + # during GC of sessions from earlier tests) can land in the same recording window. + return [x for x in recorded if "does not support speed" in str(x.message)] + + +@pytest.mark.asyncio +async def test_realtime_connect_speed_fast_no_warning_for_supported_model(): + import warnings + + from decart.models import _warned_unsupported_speeds + + _warned_unsupported_speeds.clear() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + for model in ("lucy-2.5", "lucy-latest", "lucy-vton-3.5", "lucy-vton-latest"): + url = await _connect_and_capture_url(speed="fast", model=model) + assert "&speed=fast" in url + assert _speed_warnings(w) == [] + + +@pytest.mark.asyncio +async def test_realtime_connect_speed_fast_warns_once_but_still_sends_for_unsupported_model(): + import warnings + + from decart.models import _warned_unsupported_speeds + + _warned_unsupported_speeds.clear() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + url = await _connect_and_capture_url(speed="fast", model="lucy-2.1") + assert "&speed=fast" in url + speed_warnings = _speed_warnings(w) + assert len(speed_warnings) == 1 + assert issubclass(speed_warnings[0].category, UserWarning) + assert 'Model "lucy-2.1" does not support speed="fast"' in str(speed_warnings[0].message) + + # One-shot per (model, speed): a second connect does not warn again. + await _connect_and_capture_url(speed="fast", model="lucy-2.1") + assert len(_speed_warnings(w)) == 1 + + +@pytest.mark.asyncio +async def test_realtime_reconnect_redials_with_speed_fast_preserved(): + from decart.realtime.livekit_manager import LiveKitConfiguration, LiveKitManager + + # Build the signaling URL through the real connect() path, then drive the manager's + # reconnect loop with it to prove the re-dial reuses the same URL (speed included). + ws_url = await _connect_and_capture_url(speed="fast", model="lucy-2.5") + assert "&speed=fast" in ws_url + + dialed_urls: list[str] = [] + + def fake_connection() -> MagicMock: + conn = MagicMock() + + async def _connect(url, **_kwargs): + dialed_urls.append(url) + + conn.connect = AsyncMock(side_effect=_connect) + conn.cleanup = AsyncMock() + return conn + + config = LiveKitConfiguration( + livekit_url=ws_url, + api_key="test-key", + session_id="", + fps=30, + on_remote_stream=lambda t: None, + ) + manager = LiveKitManager(config) + manager._create_connection = fake_connection # type: ignore[method-assign] + + assert await manager.connect(local_track=MagicMock()) + manager._handle_connection_state_change("connected") + assert manager.is_connected() + + # Unexpected drop -> manager schedules a reconnect and re-dials the stored URL. + manager._handle_connection_state_change("disconnected") + assert manager._reconnect_task is not None + await manager._reconnect_task + + assert len(dialed_urls) == 2 + assert dialed_urls[1] == ws_url + assert all(url.count("speed=fast") == 1 for url in dialed_urls) + + await manager.cleanup() + + @pytest.mark.asyncio async def test_realtime_connect_allows_preferred_video_codec_override(): from decart.realtime.types import RealtimeConnectOptions