Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,36 @@ async with DecartClient(api_key=os.getenv("DECART_API_KEY")) as client:
f.write(data)
```

### Realtime fast mode

Realtime sessions accept an optional `speed` on `RealtimeConnectOptions`, alongside `resolution`.
Fast mode (`speed="fast"`) serves the session from a higher-compute tier for lower latency and
higher throughput; output quality is unchanged. It is currently available for `lucy-2.5` /
`lucy-latest` and `lucy-vton-3.5` / `lucy-vton-latest`, in the US region only, and is billed at
2x the standard realtime rate for those models. Other models ignore the option (the SDK emits a
warning). Omit it (the default) for standard mode.

```python
from decart import DecartClient, models
from decart.realtime import RealtimeClient, RealtimeConnectOptions

client = DecartClient(api_key=os.getenv("DECART_API_KEY"))
realtime = await RealtimeClient.connect(
base_url=client.realtime_base_url,
api_key=client.api_key,
local_track=local_track,
options=RealtimeConnectOptions(
model=models.realtime("lucy-2.5"),
on_remote_stream=on_remote_stream,
speed="fast", # omit for standard mode
),
)
```

Each model definition lists the speed tiers it advertises via `ModelDefinition.supported_speeds`
(for example `models.realtime("lucy-2.5").supported_speeds == ("fast",)`). See the
[realtime docs](https://docs.platform.decart.ai/sdks/python) for the full realtime API.

## Development

### Setup with UV
Expand Down
9 changes: 8 additions & 1 deletion decart/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,13 @@
QueueResultError,
TokenCreateError,
)
from .models import models, ModelDefinition, CustomModelDefinition, VideoRestyleInput
from .models import (
models,
ModelDefinition,
CustomModelDefinition,
VideoRestyleInput,
RealtimeSpeed,
)
from .types import FileInput, ModelState, Prompt
from .queue import (
QueueClient,
Expand Down Expand Up @@ -71,6 +77,7 @@
"ModelDefinition",
"CustomModelDefinition",
"VideoRestyleInput",
"RealtimeSpeed",
"FileInput",
"ModelState",
"Prompt",
Expand Down
35 changes: 34 additions & 1 deletion decart/models.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import warnings
from typing import Literal, Optional, Generic, TypeVar
from typing import Any, Literal, Optional, Generic, TypeVar
from pydantic import BaseModel, Field, ConfigDict, model_validator
from .errors import ModelNotFoundError
from .types import FileInput
Expand Down Expand Up @@ -41,6 +41,10 @@
]
Model = Literal[RealTimeModels, VideoModels, ImageModels]

RealtimeSpeed = Literal["fast"]
"""Realtime speed tier. ``"fast"`` serves the session from a higher-compute tier
for lower latency and higher throughput; omit it (the default) for standard mode."""

MODEL_ALIASES: dict[str, str] = {
# Video aliases
"lucy-pro-v2v": "lucy-clip",
Expand All @@ -64,6 +68,29 @@ def _warn_deprecated(model: str) -> None:
)


_warned_unsupported_speeds: set[tuple[str, str]] = set()


def _warn_unsupported_speed(model: "ModelDefinition[Any]", speed: str) -> None:
"""Warn once per (model, speed) when a realtime speed tier the model does not
advertise is requested. The option is still sent; the server ignores it for
models without the tier."""
if speed in model.supported_speeds:
return
key = (model.name, speed)
if key in _warned_unsupported_speeds:
return
_warned_unsupported_speeds.add(key)
warnings.warn(
f'Model "{model.name}" does not support speed="{speed}"; the option is ignored by the '
"server for this model. Fast mode is currently available for lucy-2.5 / lucy-latest and "
"lucy-vton-3.5 / lucy-vton-latest only. See https://docs.platform.decart.ai/models "
"for details.",
UserWarning,
stacklevel=3,
)


# Type variable for model name
ModelT = TypeVar("ModelT", bound=str)

Expand All @@ -79,6 +106,8 @@ class ModelDefinition(DecartBaseModel, Generic[ModelT]):
width: int = Field(ge=1)
height: int = Field(ge=1)
input_schema: Optional[type[BaseModel]] = None
supported_speeds: tuple[RealtimeSpeed, ...] = ()
"""Realtime speed tiers this model advertises (empty for standard mode only)."""


# Type aliases for model definitions that support specific APIs
Expand Down Expand Up @@ -182,6 +211,7 @@ class ImageToImageInput(DecartBaseModel):
fps=30,
width=1280,
height=720,
supported_speeds=("fast",),
),
"lucy-restyle-2": ModelDefinition(
name="lucy-restyle-2",
Expand All @@ -197,6 +227,7 @@ class ImageToImageInput(DecartBaseModel):
fps=30,
width=1088,
height=624,
supported_speeds=("fast",),
),
# Server-side alias currently resolves to lucy-vton-3.5.
"lucy-vton-latest": ModelDefinition(
Expand All @@ -205,13 +236,15 @@ class ImageToImageInput(DecartBaseModel):
fps=30,
width=1280,
height=720,
supported_speeds=("fast",),
),
"lucy-vton-3.5": ModelDefinition(
name="lucy-vton-3.5",
url_path="/v1/stream",
fps=30,
width=1280,
height=720,
supported_speeds=("fast",),
),
"lucy-restyle-latest": ModelDefinition(
name="lucy-restyle-latest",
Expand Down
3 changes: 2 additions & 1 deletion decart/realtime/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
decode_subscribe_token,
)
from .messages import GenerationTickMessage
from .types import RealtimeConnectOptions, ConnectionState
from .types import RealtimeConnectOptions, ConnectionState, RealtimeSpeed

__all__ = [
"RealtimeClient",
Expand All @@ -18,4 +18,5 @@
"GenerationTickMessage",
"RealtimeConnectOptions",
"ConnectionState",
"RealtimeSpeed",
]
4 changes: 4 additions & 0 deletions decart/realtime/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from .types import ConnectionState, RealtimeConnectOptions
from ..types import FileInput
from ..errors import DecartSDKError, InvalidInputError, WebRTCError
from ..models import _warn_unsupported_speed
from ..process.request import file_input_to_bytes

if TYPE_CHECKING:
Expand Down Expand Up @@ -174,6 +175,9 @@ async def connect(
)
if options.resolution is not None:
ws_url += f"&resolution={quote(options.resolution)}"
if options.speed is not None:
_warn_unsupported_speed(options.model, options.speed)
ws_url += f"&speed={quote(options.speed)}"

config = LiveKitConfiguration(
livekit_url=ws_url,
Expand Down
10 changes: 9 additions & 1 deletion decart/realtime/types.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from typing import Literal, Callable, Optional, TYPE_CHECKING
from dataclasses import dataclass
from ..models import ModelDefinition
from ..models import ModelDefinition, RealtimeSpeed
from ..types import ModelState

if TYPE_CHECKING:
Expand All @@ -10,6 +10,8 @@
ConnectionState = Literal["connecting", "connected", "generating", "disconnected", "reconnecting"]
VideoCodec = Literal["h264", "vp9"]

__all__ = ["ConnectionState", "VideoCodec", "RealtimeSpeed", "RealtimeConnectOptions"]


@dataclass
class RealtimeConnectOptions:
Expand All @@ -18,3 +20,9 @@ class RealtimeConnectOptions:
initial_state: Optional[ModelState] = None
resolution: Optional[Literal["720p", "1080p"]] = None
preferred_video_codec: VideoCodec = "h264"
speed: Optional[RealtimeSpeed] = None
"""Realtime speed tier. ``"fast"`` serves the session from a higher-compute tier for
lower latency and higher throughput; output quality is unchanged. Currently available
for lucy-2.5 / lucy-latest and lucy-vton-3.5 / lucy-vton-latest, in the US region only,
and billed at 2x the standard realtime rate for those models. Other models ignore the
option. Omit it (the default) for standard mode."""
7 changes: 7 additions & 0 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ pip install decart[realtime]
- **`realtime_synthetic.py`** - Publish synthetic colored frames through LiveKit
- **`realtime_file.py`** - Publish frames from a video file through LiveKit

Fast mode: pass `speed="fast"` on `RealtimeConnectOptions` to serve the session from a
higher-compute tier for lower latency and higher throughput (output quality is unchanged). It is
currently available for `lucy-2.5` / `lucy-latest` and `lucy-vton-3.5` / `lucy-vton-latest`, in
the US region only, and is billed at 2x the standard realtime rate for those models. Other models
ignore the option. Omit it (the default) for standard mode. The playground exposes it as
`python playground/playground.py --model lucy-2.5 --speed fast`.

### Running Examples

`process_image.py` uses the bundled `examples/files/image.png` asset.
Expand Down
4 changes: 4 additions & 0 deletions examples/realtime_synthetic.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ def on_error(error):
),
image=Path("examples/files/image.png"),
),
# Fast mode: uncomment with a supporting model (lucy-2.5 / lucy-latest,
# lucy-vton-3.5 / lucy-vton-latest). US region only, billed at 2x the
# standard realtime rate; other models ignore it. Omit for standard mode.
# speed="fast",
),
)

Expand Down
14 changes: 13 additions & 1 deletion playground/playground.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def _check_deps() -> None:
from livekit import rtc # noqa: E402

from decart import DecartClient, models # noqa: E402
from decart.models import RealTimeModels # noqa: E402
from decart.models import RealtimeSpeed, RealTimeModels # noqa: E402
from decart.realtime.client import RealtimeClient # noqa: E402
from decart.realtime.types import RealtimeConnectOptions # noqa: E402
from decart.types import ModelState, Prompt # noqa: E402
Expand Down Expand Up @@ -119,6 +119,15 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--api-key", "-k", help="API key (or set DECART_API_KEY)")
parser.add_argument("--image", "-i", help="Optional reference image")
parser.add_argument("--prompt", "-p", help="Initial prompt text")
parser.add_argument(
"--speed",
choices=["fast"],
help=(
"Realtime speed tier. 'fast' uses a higher-compute tier for lower latency "
"(lucy-2.5 / lucy-vton-3.5 and their -latest aliases only, US region, billed at 2x). "
"Omit for standard mode."
),
)
parser.add_argument("--camera", "-c", type=int, default=0, help="Camera device index")
parser.add_argument("--no-local", action="store_true", help="Hide local camera feed")
parser.add_argument("--verbose", "-v", action="store_true", help="Enable debug logging")
Expand Down Expand Up @@ -156,8 +165,10 @@ async def run() -> None:

model_name = args.model or select_model_interactive()
model = models.realtime(cast(RealTimeModels, model_name))
speed = cast(Optional[RealtimeSpeed], args.speed)
print(f"\n Model : {model_name}")
print(f" Res : {model.width}x{model.height} @ {model.fps}fps")
print(f" Speed : {speed or 'standard'}")

if args.image and not Path(args.image).exists():
print(f"\nError: Image not found: {args.image}")
Expand Down Expand Up @@ -223,6 +234,7 @@ def _read_prompts() -> None:
model=model,
on_remote_stream=on_remote_stream,
initial_state=initial_state,
speed=speed,
),
)
realtime.on("connection_change", on_connection_change)
Expand Down
34 changes: 32 additions & 2 deletions tests/test_models.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import warnings
import pytest
from decart import models, DecartSDKError, ModelDefinition
from decart.models import _warned_aliases
from decart import models, DecartSDKError, ModelDefinition, RealtimeSpeed
from decart.models import _MODELS, _warned_aliases


def test_canonical_realtime_models() -> None:
Expand Down Expand Up @@ -183,6 +183,35 @@ def test_latest_aliases_no_deprecation_warning() -> None:
assert len(w) == 0


FAST_REALTIME_MODELS = {"lucy-2.5", "lucy-latest", "lucy-vton-3.5", "lucy-vton-latest"}


def test_realtime_fast_speed_capability_pinned_to_lucy_2_5_and_vton_3_5() -> None:
# Matches the JS SDK registry: only these realtime entries advertise speed="fast".
fast_models = {
name for name, model in _MODELS["realtime"].items() if "fast" in model.supported_speeds
}
assert fast_models == FAST_REALTIME_MODELS

for name, model in _MODELS["realtime"].items():
if name in FAST_REALTIME_MODELS:
assert model.supported_speeds == ("fast",)
else:
assert model.supported_speeds == ()


def test_non_realtime_models_advertise_no_speeds() -> None:
for surface in ("video", "image"):
for model in _MODELS[surface].values():
assert model.supported_speeds == ()


def test_realtime_speed_literal_is_fast_only() -> None:
from typing import get_args

assert get_args(RealtimeSpeed) == ("fast",)


def test_custom_model_definition_allows_arbitrary_model_names() -> None:
model = ModelDefinition(
name="lucy_2_rt_preview",
Expand All @@ -194,6 +223,7 @@ def test_custom_model_definition_allows_arbitrary_model_names() -> None:

assert model.name == "lucy_2_rt_preview"
assert model.input_schema is None
assert model.supported_speeds == ()


def test_invalid_model() -> None:
Expand Down
Loading
Loading