Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ Changes are grouped as follows
* In the `unstable` package: New `Extractor.get_log_upload_data_set_id()` hook that tells the built-in `fetch_logs` action which data set to upload log files into. Resolution order: integration "data domains" field (not available yet, marked TODO), then the extractor's own configured data set, then no data set. The default uses `application_config.data_set` when the config derives from `WithDataSetId` and otherwise returns `None`; extractors that keep their data set elsewhere (for example the File and DB extractors) should override it in their own repositories.

### Changed
* In the `unstable` package: The maximum number of available actions sent at startup is raised from 100 to 250, so an extractor with up to 100 actionable tasks (a start and a stop action each) plus custom actions can register. Requires an Odin version with the same limit (EDG-1049).

* In the `unstable` package: `fetch_logs` now uploads a new copy of the log files on every invocation instead of overwriting: the file external ID is now `extractor-logs-<integration>-<date>-<action external id>` (the `extractor-logs-<integration>-<date>` prefix is unchanged). The per-file status `skipped` for a date without a log file is now called `missing`, matching the existing `missing_files` counter (all counter keys are unchanged). When no file was uploaded at all the action now ends as `completed_with_errors` with the message "No log files were uploaded ..." instead of `succeeded`. Custom actions can end as `completed_with_errors` with `ActionContext.set_result(..., completed_with_errors=True)`.
* In the `unstable` package: `Stop <task>` actions now stay `running` until the task has actually ended and only then report `succeeded`. If the task has not ended within 300 seconds (`STOP_TASK_ACTION_TIMEOUT_SECONDS`) the action is reported as `failed` ("Task '<name>' did not stop within 300 seconds"); a waiting Stop is reported as `canceled` on extractor shutdown.

Expand Down
4 changes: 3 additions & 1 deletion cognite/extractorutils/unstable/core/_dto.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,9 @@ def validate_status(cls, v: ActionStatus) -> ActionStatus:
return v


AvailableActionList = Annotated[list[AvailableActionWrite], Len(min_length=0, max_length=100)]
# Each actionable task registers a start and a stop action; keep in sync with the Odin limit.
MAX_AVAILABLE_ACTIONS = 250
AvailableActionList = Annotated[list[AvailableActionWrite], Len(min_length=0, max_length=MAX_AVAILABLE_ACTIONS)]
ActionUpdateList = Annotated[list[ActionUpdate], Len(min_length=0, max_length=100)]


Expand Down
20 changes: 19 additions & 1 deletion tests/test_unstable/test_action_registration.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
from unittest.mock import MagicMock

import pytest
from pydantic import ValidationError

from cognite.extractorutils.unstable.core._dto import ActionType, StartupRequest
from cognite.extractorutils.unstable.core._dto import MAX_AVAILABLE_ACTIONS, ActionType, StartupRequest
from cognite.extractorutils.unstable.core.actions import CustomAction
from cognite.extractorutils.unstable.core.base import Extractor, FullConfig
from cognite.extractorutils.unstable.core.tasks import ContinuousTask, ScheduledTask, StartupTask, TaskContext
Expand Down Expand Up @@ -212,3 +213,20 @@ def target(_: TaskContext) -> None:
time.sleep(0.005)

assert extractor._running_task_tokens.get("the-task") is replacement


def test_startup_request_accepts_two_actions_per_task_at_odin_task_limit() -> None:
extractor = _make_extractor()
for i in range(100):
extractor.add_task(ScheduledTask.from_interval(interval="1h", name=f"task-{i}", target=lambda _: None))
actions = _startup_request(extractor).available_actions
assert actions is not None
assert len(actions) >= 200


def test_startup_request_rejects_more_than_max_available_actions() -> None:
extractor = _make_extractor()
for i in range(MAX_AVAILABLE_ACTIONS):
extractor.add_task(ScheduledTask.from_interval(interval="1h", name=f"task-{i}", target=lambda _: None))
with pytest.raises(ValidationError):
_startup_request(extractor)
7 changes: 5 additions & 2 deletions tests/test_unstable/test_dto.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from pydantic import ValidationError

from cognite.extractorutils.unstable.core._dto import (
MAX_AVAILABLE_ACTIONS,
MAX_MESSAGE_LENGTH,
MAX_METADATA_VALUE_BYTES,
Action,
Expand Down Expand Up @@ -99,11 +100,13 @@ def test_startup_request_available_actions_none_excluded() -> None:


def test_startup_request_available_actions_max_length_enforced() -> None:
with pytest.raises(ValidationError, match="at most 100"):
with pytest.raises(ValidationError, match=f"at most {MAX_AVAILABLE_ACTIONS}"):
StartupRequest(
external_id="x",
extractor=ExtractorInfo(external_id="x"),
available_actions=[AvailableActionWrite(name=f"a{i}", type=ActionType.custom) for i in range(101)],
available_actions=[
AvailableActionWrite(name=f"a{i}", type=ActionType.custom) for i in range(MAX_AVAILABLE_ACTIONS + 1)
],
)


Expand Down