diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ef89fa0..6c96ae59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ Changes are grouped as follows * In the `unstable` package: New `Extractor.get_log_upload_data_set_id()` hook that tells the built-in `fetch_logs` action which data set to upload log files into. Resolution order: integration "data domains" field (not available yet, marked TODO), then the extractor's own configured data set, then no data set. The default uses `application_config.data_set` when the config derives from `WithDataSetId` and otherwise returns `None`; extractors that keep their data set elsewhere (for example the File and DB extractors) should override it in their own repositories. ### Changed +* In the `unstable` package: The maximum number of available actions sent at startup is raised from 100 to 250, so an extractor with up to 100 actionable tasks (a start and a stop action each) plus custom actions can register. Requires an Odin version with the same limit (EDG-1049). + * In the `unstable` package: `fetch_logs` now uploads a new copy of the log files on every invocation instead of overwriting: the file external ID is now `extractor-logs---` (the `extractor-logs--` prefix is unchanged). The per-file status `skipped` for a date without a log file is now called `missing`, matching the existing `missing_files` counter (all counter keys are unchanged). When no file was uploaded at all the action now ends as `completed_with_errors` with the message "No log files were uploaded ..." instead of `succeeded`. Custom actions can end as `completed_with_errors` with `ActionContext.set_result(..., completed_with_errors=True)`. * In the `unstable` package: `Stop ` actions now stay `running` until the task has actually ended and only then report `succeeded`. If the task has not ended within 300 seconds (`STOP_TASK_ACTION_TIMEOUT_SECONDS`) the action is reported as `failed` ("Task '' did not stop within 300 seconds"); a waiting Stop is reported as `canceled` on extractor shutdown. diff --git a/cognite/extractorutils/unstable/core/_dto.py b/cognite/extractorutils/unstable/core/_dto.py index 09ad510f..c2b7b2c1 100644 --- a/cognite/extractorutils/unstable/core/_dto.py +++ b/cognite/extractorutils/unstable/core/_dto.py @@ -196,7 +196,9 @@ def validate_status(cls, v: ActionStatus) -> ActionStatus: return v -AvailableActionList = Annotated[list[AvailableActionWrite], Len(min_length=0, max_length=100)] +# Each actionable task registers a start and a stop action; keep in sync with the Odin limit. +MAX_AVAILABLE_ACTIONS = 250 +AvailableActionList = Annotated[list[AvailableActionWrite], Len(min_length=0, max_length=MAX_AVAILABLE_ACTIONS)] ActionUpdateList = Annotated[list[ActionUpdate], Len(min_length=0, max_length=100)] diff --git a/tests/test_unstable/test_action_registration.py b/tests/test_unstable/test_action_registration.py index 153aaf8b..1ff1f95c 100644 --- a/tests/test_unstable/test_action_registration.py +++ b/tests/test_unstable/test_action_registration.py @@ -4,8 +4,9 @@ from unittest.mock import MagicMock import pytest +from pydantic import ValidationError -from cognite.extractorutils.unstable.core._dto import ActionType, StartupRequest +from cognite.extractorutils.unstable.core._dto import MAX_AVAILABLE_ACTIONS, ActionType, StartupRequest from cognite.extractorutils.unstable.core.actions import CustomAction from cognite.extractorutils.unstable.core.base import Extractor, FullConfig from cognite.extractorutils.unstable.core.tasks import ContinuousTask, ScheduledTask, StartupTask, TaskContext @@ -212,3 +213,20 @@ def target(_: TaskContext) -> None: time.sleep(0.005) assert extractor._running_task_tokens.get("the-task") is replacement + + +def test_startup_request_accepts_two_actions_per_task_at_odin_task_limit() -> None: + extractor = _make_extractor() + for i in range(100): + extractor.add_task(ScheduledTask.from_interval(interval="1h", name=f"task-{i}", target=lambda _: None)) + actions = _startup_request(extractor).available_actions + assert actions is not None + assert len(actions) >= 200 + + +def test_startup_request_rejects_more_than_max_available_actions() -> None: + extractor = _make_extractor() + for i in range(MAX_AVAILABLE_ACTIONS): + extractor.add_task(ScheduledTask.from_interval(interval="1h", name=f"task-{i}", target=lambda _: None)) + with pytest.raises(ValidationError): + _startup_request(extractor) diff --git a/tests/test_unstable/test_dto.py b/tests/test_unstable/test_dto.py index 64e975db..63256bf5 100644 --- a/tests/test_unstable/test_dto.py +++ b/tests/test_unstable/test_dto.py @@ -2,6 +2,7 @@ from pydantic import ValidationError from cognite.extractorutils.unstable.core._dto import ( + MAX_AVAILABLE_ACTIONS, MAX_MESSAGE_LENGTH, MAX_METADATA_VALUE_BYTES, Action, @@ -99,11 +100,13 @@ def test_startup_request_available_actions_none_excluded() -> None: def test_startup_request_available_actions_max_length_enforced() -> None: - with pytest.raises(ValidationError, match="at most 100"): + with pytest.raises(ValidationError, match=f"at most {MAX_AVAILABLE_ACTIONS}"): StartupRequest( external_id="x", extractor=ExtractorInfo(external_id="x"), - available_actions=[AvailableActionWrite(name=f"a{i}", type=ActionType.custom) for i in range(101)], + available_actions=[ + AvailableActionWrite(name=f"a{i}", type=ActionType.custom) for i in range(MAX_AVAILABLE_ACTIONS + 1) + ], )