From 8b203534c3bbcf08ce9224b86c8d957d07a0989a Mon Sep 17 00:00:00 2001 From: Sarav Date: Mon, 5 Oct 2026 11:27:47 +0530 Subject: [PATCH 1/3] fix: [AI-9453] parse dbt-core 1.11+ manifests and stop downgrading dbt deps - Add `javascript` to manifest v12 `SupportedLanguage`; dbt-core 1.11+ ships `materialization_function_default` with it, so every 1.11+ manifest failed - `disabled` fallback now nulls the field instead of dropping it; v11/v12 declare it required, so the retry always failed and masked the real error - Relax `click` and `python-dotenv` pins to `<9.0` / `<2.0`; the old `~=` pins downgraded packages dbt-core 1.11+ requires - Add manifest v12 regression tests Co-Authored-By: Claude Opus 5.5 (1M context) --- setup.py | 4 +- src/vendor/dbt_artifacts_parser/parser.py | 7 +- .../parsers/manifest/manifest_v12.py | 2 + tests/test_vendor/test_manifest_v12.py | 75 +++++++++++++++++++ 4 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 tests/test_vendor/test_manifest_v12.py diff --git a/setup.py b/setup.py index ab213c0..1824c01 100755 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ def read(*names, **kwargs): ], python_requires=">=3.8", install_requires=[ - "click~=8.1.7", + "click>=8.1.7,<9.0", "pydantic >=2.0,<3.0", "ruamel.yaml~=0.18.6", "tabulate~=0.9.0", @@ -70,7 +70,7 @@ def read(*names, **kwargs): "sqlglot[c]==30.11.0", "mcp>=1.9.0,<2.0.0", "pyperclip~=1.8.2", - "python-dotenv~=1.0.0", + "python-dotenv>=1.0.0,<2.0", ], extras_require={ # eg: diff --git a/src/vendor/dbt_artifacts_parser/parser.py b/src/vendor/dbt_artifacts_parser/parser.py index 9e12d72..eaa61ef 100644 --- a/src/vendor/dbt_artifacts_parser/parser.py +++ b/src/vendor/dbt_artifacts_parser/parser.py @@ -86,12 +86,13 @@ def parse_catalog_v1(catalog: dict) -> CatalogV1: # manifest # def _strip_unused_fields(manifest: dict) -> dict: - """Remove fields that have strict discriminated unions but are unused downstream. + """Null out fields that have strict discriminated unions but are unused downstream. These fields (e.g. `disabled`) use complex Pydantic unions that break when - dbt Cloud changes its schema, but our wrappers never read them. + dbt Cloud changes its schema, but our wrappers never read them. They are + set to None rather than dropped because v11+ declare them required (but nullable). """ - return {k: v for k, v in manifest.items() if k not in _UNUSED_STRICT_FIELDS} + return {k: (None if k in _UNUSED_STRICT_FIELDS else v) for k, v in manifest.items()} def _try_parse_manifest(manifest: dict, model_class): diff --git a/src/vendor/dbt_artifacts_parser/parsers/manifest/manifest_v12.py b/src/vendor/dbt_artifacts_parser/parsers/manifest/manifest_v12.py index e6ded95..934ea0b 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/manifest/manifest_v12.py +++ b/src/vendor/dbt_artifacts_parser/parsers/manifest/manifest_v12.py @@ -1640,6 +1640,8 @@ class Argument(BaseParserModel): class SupportedLanguage(Enum): python = "python" sql = "sql" + # dbt-core 1.11+ ships `materialization_function_default` supporting UDFs in JavaScript + javascript = "javascript" class Macros(BaseParserModel): diff --git a/tests/test_vendor/test_manifest_v12.py b/tests/test_vendor/test_manifest_v12.py new file mode 100644 index 0000000..93795d6 --- /dev/null +++ b/tests/test_vendor/test_manifest_v12.py @@ -0,0 +1,75 @@ +"""Tests for manifest v12 parsing against manifests produced by dbt-core 1.11+. + +dbt-core 1.11 added the built-in ``materialization_function_default`` macro, whose +``supported_languages`` include ``"javascript"``. The v12 ``SupportedLanguage`` enum +only allowed ``python``/``sql``, so EVERY manifest from dbt-core 1.11+ failed to parse +and project governance broke for all users on current dbt. + +The failure was masked by the ``disabled`` fallback: it dropped ``disabled`` from the +manifest, but v12 declares that field required (nullable), so the retry always raised +``disabled: Field required`` instead of the real error. +""" +import copy +import json +from pathlib import Path + +import pytest + +from vendor.dbt_artifacts_parser.parser import parse_manifest +from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12 + +MANIFEST_V12 = Path(__file__).parent.parent / "data" / "manifest_v12.json" +FUNCTION_MACRO_ID = "macro.dbt.materialization_function_default" + + +@pytest.fixture +def manifest() -> dict: + with MANIFEST_V12.open() as f: + return json.load(f) + + +def _with_function_macro(manifest: dict) -> dict: + """Add the macro dbt-core 1.11+ ships, cloned from an existing macro in the fixture.""" + manifest = copy.deepcopy(manifest) + macro = copy.deepcopy(next(iter(manifest["macros"].values()))) + macro.update( + unique_id=FUNCTION_MACRO_ID, + name="materialization_function_default", + supported_languages=["sql", "python", "javascript"], + ) + manifest["macros"][FUNCTION_MACRO_ID] = macro + return manifest + + +def test_parses_javascript_supported_language(manifest): + """GIVEN a dbt-core 1.11+ manifest containing a macro that supports JavaScript + WHEN it is parsed + THEN parsing succeeds and the language is preserved.""" + parsed = parse_manifest(_with_function_macro(manifest)) + + assert isinstance(parsed, ManifestV12) + languages = [lang.value for lang in parsed.macros[FUNCTION_MACRO_ID].supported_languages] + assert languages == ["sql", "python", "javascript"] + + +def test_unparseable_disabled_falls_back_to_none(manifest): + """GIVEN a manifest whose ``disabled`` section does not match the strict schema + WHEN it is parsed + THEN the fallback nulls ``disabled`` (required in v12) and parsing succeeds.""" + manifest["disabled"] = {"model.proj.broken": [{"resource_type": "not-a-real-type"}]} + + parsed = parse_manifest(manifest) + + assert isinstance(parsed, ManifestV12) + assert parsed.disabled is None + + +def test_unrelated_errors_still_raise(manifest): + """GIVEN a manifest that is invalid outside the ``disabled`` section + WHEN it is parsed + THEN the original validation error surfaces instead of being masked.""" + manifest = _with_function_macro(manifest) + manifest["macros"][FUNCTION_MACRO_ID]["supported_languages"] = ["cobol"] + + with pytest.raises(ValueError, match="supported_languages"): + parse_manifest(manifest) From 40f62c3b9bedfe487f91d490c5ce4a3864015017 Mon Sep 17 00:00:00 2001 From: Sarav Date: Mon, 5 Oct 2026 12:04:08 +0530 Subject: [PATCH 2/3] fix: [AI-9453] accept `javascript` when wrapping project macros `AltimateSupportedLanguage` aliased the manifest v11 enum (python/sql only), so `ManifestV12Wrapper._get_macro` still raised `ValueError` for any project-owned macro supporting JavaScript (e.g. a custom UDF materialization). Alias the v12 enum, a superset of v11, and add a wrapper-level test. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../core/platforms/dbt/schemas/manifest.py | 2 +- tests/test_vendor/test_manifest_v12.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/datapilot/core/platforms/dbt/schemas/manifest.py b/src/datapilot/core/platforms/dbt/schemas/manifest.py index 3a4b33f..45e93b6 100644 --- a/src/datapilot/core/platforms/dbt/schemas/manifest.py +++ b/src/datapilot/core/platforms/dbt/schemas/manifest.py @@ -18,8 +18,8 @@ from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v9 import ManifestV9 from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v10 import ManifestV10 from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11 -from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v11 import SupportedLanguage from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12 +from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v12 import SupportedLanguage class DBTVersion(BaseModel): diff --git a/tests/test_vendor/test_manifest_v12.py b/tests/test_vendor/test_manifest_v12.py index 93795d6..ed09b79 100644 --- a/tests/test_vendor/test_manifest_v12.py +++ b/tests/test_vendor/test_manifest_v12.py @@ -15,6 +15,7 @@ import pytest +from datapilot.core.platforms.dbt.factory import DBTFactory from vendor.dbt_artifacts_parser.parser import parse_manifest from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12 @@ -52,6 +53,20 @@ def test_parses_javascript_supported_language(manifest): assert languages == ["sql", "python", "javascript"] +def test_wraps_project_macro_supporting_javascript(manifest): + """GIVEN a project-owned macro (e.g. a custom UDF materialization) supporting JavaScript + WHEN the parsed manifest is wrapped for insights + THEN the macro's languages convert without a ValueError.""" + manifest = _with_function_macro(manifest) + project = manifest["metadata"]["project_name"] + manifest["macros"][FUNCTION_MACRO_ID]["package_name"] = project + + macros = DBTFactory.get_manifest_wrapper(parse_manifest(manifest)).get_macros() + + languages = [lang.value for lang in macros[FUNCTION_MACRO_ID].supported_languages] + assert languages == ["sql", "python", "javascript"] + + def test_unparseable_disabled_falls_back_to_none(manifest): """GIVEN a manifest whose ``disabled`` section does not match the strict schema WHEN it is parsed From e7d21637a7e61855ab4682b519072f9984488b3b Mon Sep 17 00:00:00 2001 From: Sarav Date: Mon, 5 Oct 2026 19:05:26 +0530 Subject: [PATCH 3/3] fix: [AI-9453] convert macro languages by value in v10/v11 wrappers - v10/v11 wrappers passed vendored `SupportedLanguage` members straight into `AltimateManifestMacroNode`, now typed with the v12 enum; Pydantic v2 rejects members of another Enum class, so `get_macros()` raised `ValidationError` for any v10/v11 project-owned macro declaring `supported_languages` (v11 regressed in this PR, v10 already failed on `main`) - `_try_parse_manifest` raises the original error when the `disabled`-nulled retry also fails, and logs the retry at debug level - Rename `_strip_unused_fields` to `_null_unused_fields` - Add v10/v11 wrapper tests and replace the masking test with one that can detect masking Co-Authored-By: Claude Opus 5.5 (1M context) --- .../dbt/wrappers/manifest/v10/wrapper.py | 5 ++- .../dbt/wrappers/manifest/v11/wrapper.py | 5 ++- src/vendor/dbt_artifacts_parser/parser.py | 15 +++++---- .../dbt/test_manifest_wrapper_macros.py | 33 +++++++++++++++++++ tests/test_vendor/test_manifest_v12.py | 19 +++++++---- 5 files changed, 62 insertions(+), 15 deletions(-) create mode 100644 tests/core/platform/dbt/test_manifest_wrapper_macros.py diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py index 842f9ad..2bb4d56 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py @@ -28,6 +28,7 @@ from datapilot.core.platforms.dbt.schemas.manifest import AltimateSeedConfig from datapilot.core.platforms.dbt.schemas.manifest import AltimateSeedNode from datapilot.core.platforms.dbt.schemas.manifest import AltimateSourceConfig +from datapilot.core.platforms.dbt.schemas.manifest import AltimateSupportedLanguage from datapilot.core.platforms.dbt.schemas.manifest import AltimateTestConfig from datapilot.core.platforms.dbt.schemas.manifest import AltimateTestMetadata from datapilot.core.platforms.dbt.wrappers.manifest.v10.schemas import TEST_TYPE_TO_NODE_MAP @@ -181,7 +182,9 @@ def _get_macro(self, macro: MacroNode) -> AltimateManifestMacroNode: patch_path=macro.patch_path, arguments=[AltimateMacroArgument(**arg.model_dump()) for arg in macro.arguments] if macro.arguments else None, created_at=macro.created_at, - supported_languages=macro.supported_languages, + supported_languages=[AltimateSupportedLanguage(lang.value) for lang in macro.supported_languages] + if macro.supported_languages + else None, ) def _get_exposure(self, exposure: ExposureNode) -> AltimateManifestExposureNode: diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py index b9610b3..ddd25f3 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py @@ -28,6 +28,7 @@ from datapilot.core.platforms.dbt.schemas.manifest import AltimateSeedConfig from datapilot.core.platforms.dbt.schemas.manifest import AltimateSeedNode from datapilot.core.platforms.dbt.schemas.manifest import AltimateSourceConfig +from datapilot.core.platforms.dbt.schemas.manifest import AltimateSupportedLanguage from datapilot.core.platforms.dbt.schemas.manifest import AltimateTestConfig from datapilot.core.platforms.dbt.schemas.manifest import AltimateTestMetadata from datapilot.core.platforms.dbt.wrappers.manifest.v11.schemas import TEST_TYPE_TO_NODE_MAP @@ -181,7 +182,9 @@ def _get_macro(self, macro: MacroNode) -> AltimateManifestMacroNode: patch_path=macro.patch_path, arguments=[AltimateMacroArgument(**arg.model_dump()) for arg in macro.arguments] if macro.arguments else None, created_at=macro.created_at, - supported_languages=macro.supported_languages, + supported_languages=[AltimateSupportedLanguage(lang.value) for lang in macro.supported_languages] + if macro.supported_languages + else None, ) def _get_exposure(self, exposure: ExposureNode) -> AltimateManifestExposureNode: diff --git a/src/vendor/dbt_artifacts_parser/parser.py b/src/vendor/dbt_artifacts_parser/parser.py index eaa61ef..c094df6 100644 --- a/src/vendor/dbt_artifacts_parser/parser.py +++ b/src/vendor/dbt_artifacts_parser/parser.py @@ -85,7 +85,7 @@ def parse_catalog_v1(catalog: dict) -> CatalogV1: # # manifest # -def _strip_unused_fields(manifest: dict) -> dict: +def _null_unused_fields(manifest: dict) -> dict: """Null out fields that have strict discriminated unions but are unused downstream. These fields (e.g. `disabled`) use complex Pydantic unions that break when @@ -96,15 +96,18 @@ def _strip_unused_fields(manifest: dict) -> dict: def _try_parse_manifest(manifest: dict, model_class): - """Attempt to parse manifest, falling back to stripping unused fields on failure.""" + """Attempt to parse manifest, falling back to nulling unused fields on failure. + + If the fallback also fails, the original error is raised so it is not masked. + """ try: return model_class(**manifest) - except Exception: - stripped = _strip_unused_fields(manifest) + except Exception as original: + logger.debug("Manifest parse failed; retrying with %s nulled", sorted(_UNUSED_STRICT_FIELDS), exc_info=True) try: - return model_class(**stripped) + return model_class(**_null_unused_fields(manifest)) except Exception: - raise + raise original def parse_manifest( diff --git a/tests/core/platform/dbt/test_manifest_wrapper_macros.py b/tests/core/platform/dbt/test_manifest_wrapper_macros.py new file mode 100644 index 0000000..912ffdf --- /dev/null +++ b/tests/core/platform/dbt/test_manifest_wrapper_macros.py @@ -0,0 +1,33 @@ +"""Tests for converting project-owned macro `supported_languages` in the manifest wrappers. + +`AltimateManifestMacroNode.supported_languages` is typed with the v12 `SupportedLanguage` +enum. The v10/v11 wrappers passed their own vendored enum members through unchanged, and +Pydantic v2 rejects members of a different Enum class even when the value matches. So +`get_macros()`, which `DBTInsightGenerator` always calls, failed for any v10/v11 project +with a custom materialization. +""" +import json +from pathlib import Path + +import pytest + +from datapilot.core.platforms.dbt.factory import DBTFactory +from vendor.dbt_artifacts_parser.parser import parse_manifest + +DATA = Path(__file__).parent.parent.parent.parent / "data" + + +@pytest.mark.parametrize("fixture", ["manifest_v10.json", "manifest_v11.json"]) +def test_wraps_project_macro_with_supported_languages(fixture): + """GIVEN a v10/v11 manifest whose project owns a macro declaring `supported_languages` + WHEN the parsed manifest is wrapped for insights + THEN the languages convert by value without a ValidationError.""" + with (DATA / fixture).open() as f: + manifest = json.load(f) + macro_id = next(k for k, m in manifest["macros"].items() if m.get("supported_languages")) + manifest["macros"][macro_id]["package_name"] = manifest["metadata"]["project_name"] + manifest["macros"][macro_id]["supported_languages"] = ["sql"] + + macros = DBTFactory.get_manifest_wrapper(parse_manifest(manifest)).get_macros() + + assert [lang.value for lang in macros[macro_id].supported_languages] == ["sql"] diff --git a/tests/test_vendor/test_manifest_v12.py b/tests/test_vendor/test_manifest_v12.py index ed09b79..2911ff7 100644 --- a/tests/test_vendor/test_manifest_v12.py +++ b/tests/test_vendor/test_manifest_v12.py @@ -16,6 +16,7 @@ import pytest from datapilot.core.platforms.dbt.factory import DBTFactory +from vendor.dbt_artifacts_parser.parser import _try_parse_manifest from vendor.dbt_artifacts_parser.parser import parse_manifest from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12 @@ -79,12 +80,16 @@ def test_unparseable_disabled_falls_back_to_none(manifest): assert parsed.disabled is None -def test_unrelated_errors_still_raise(manifest): - """GIVEN a manifest that is invalid outside the ``disabled`` section +def test_failed_fallback_raises_the_original_error(): + """GIVEN a manifest whose first parse and `disabled`-nulled retry fail with different errors WHEN it is parsed - THEN the original validation error surfaces instead of being masked.""" - manifest = _with_function_macro(manifest) - manifest["macros"][FUNCTION_MACRO_ID]["supported_languages"] = ["cobol"] + THEN the original error is raised, not the retry's.""" + + class Model: + def __init__(self, **manifest): + if manifest["disabled"] is None: + raise ValueError("retry error") + raise ValueError("original error") - with pytest.raises(ValueError, match="supported_languages"): - parse_manifest(manifest) + with pytest.raises(ValueError, match="^original error$"): + _try_parse_manifest({"disabled": {}}, Model)