From ad36bb9e115f01c51770bad72808bf6f899c3ecb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:17:07 +0400 Subject: [PATCH 01/18] feat: implement API v2 policy migration --- .github/copilot-instructions.md | 3 + AGENTS.md | 3 + CLAUDE.md | 3 + docs/engineering/migration-contracts.md | 41 +- docs/engineering/skills/README.md | 2 + docs/engineering/skills/api-routes.md | 105 ++++ docs/engineering/skills/testing.md | 60 +++ docs/generated/migration_contracts.json | 182 ++++++- docs/migration/stage-10-v2-policies.md | 127 +++++ ...8f553ca5_add_saved_policy_mirror_events.py | 103 ++++ .../711ec2f0a5a5_migrate_v2_policies.py | 395 +++++++++++++++ ...a49_track_saved_policy_mirror_revisions.py | 49 ++ policyengine_api/asgi_factory.py | 21 + policyengine_api/data/v1_models.py | 70 +++ .../data/v2/catalog/publication.py | 2 +- policyengine_api/data/v2/database.py | 8 +- policyengine_api/data/v2/models/__init__.py | 6 + .../data/v2/models/associations.py | 51 +- policyengine_api/data/v2/models/metadata.py | 15 +- policyengine_api/data/v2/models/policies.py | 56 ++- .../data/v2/models/policy_mappings.py | 103 ++++ policyengine_api/data/v2/models/users.py | 5 - policyengine_api/data/v2/policies/__init__.py | 1 + .../data/v2/policies/api_schemas.py | 125 +++++ .../data/v2/policies/canonicalization.py | 107 ++++ policyengine_api/data/v2/policies/catalog.py | 77 +++ policyengine_api/data/v2/policies/legacy.py | 286 +++++++++++ .../data/v2/policies/persistence.py | 158 ++++++ policyengine_api/data/v2/policies/query.py | 145 ++++++ policyengine_api/data/v2/policies/schemas.py | 138 ++++++ policyengine_api/data/v2/policies/service.py | 109 +++++ .../data/v2/policy_migration_qualification.py | 177 +++++++ policyengine_api/data/v2/settings.py | 10 +- .../data/v2/user_policies/__init__.py | 1 + .../data/v2/user_policies/api_schemas.py | 118 +++++ .../data/v2/user_policies/legacy.py | 278 +++++++++++ .../data/v2/user_policies/persistence.py | 92 ++++ .../data/v2/user_policies/query.py | 115 +++++ .../data/v2/user_policies/schemas.py | 34 ++ .../data/v2/user_policies/service.py | 123 +++++ .../fastapi_routes/dependencies.py | 47 ++ .../fastapi_routes/query_parameters.py | 66 +++ .../fastapi_routes/v2_metadata.py | 6 + .../fastapi_routes/v2_policies.py | 197 ++++++++ .../fastapi_routes/v2_user_policies.py | 203 ++++++++ policyengine_api/migration_flags.py | 22 + policyengine_api/migration_logging.py | 27 +- policyengine_api/migration_registry.py | 2 +- policyengine_api/query_parameters.py | 203 ++++++++ policyengine_api/readiness.py | 31 +- policyengine_api/routes/policy_routes.py | 272 +++++++++-- policyengine_api/services/policy_mirroring.py | 146 ++++++ policyengine_api/services/policy_service.py | 64 ++- .../services/user_policy_mirroring.py | 301 ++++++++++++ .../services/user_policy_service.py | 384 +++++++++++++-- scripts/guards/migration_contracts.py | 7 +- scripts/qualify_v2_policy_migration.py | 8 + tests/contract/registry.py | 171 +++++++ .../test_app_v2_workflow_contracts.py | 18 +- .../contract/test_policy_v2_compatibility.py | 58 +++ tests/contract/test_v1_route_contracts.py | 110 +++++ .../test_alembic_mysql_lifecycle.py | 10 +- .../integration/test_alembic_v2_lifecycle.py | 33 +- .../integration/test_v1_policy_dual_write.py | 259 ++++++++++ .../test_v1_user_policy_dual_write.py | 386 +++++++++++++++ .../integration/test_v2_policy_persistence.py | 455 ++++++++++++++++++ .../test_v2_user_policy_mirroring.py | 355 ++++++++++++++ tests/unit/data/test_v1_models.py | 9 + .../routes/test_migration_context_logging.py | 44 ++ .../routes/test_policy_dual_write_routes.py | 207 ++++++++ .../test_user_policy_dual_write_routes.py | 447 +++++++++++++++++ tests/unit/services/test_policy_mirroring.py | 103 ++++ tests/unit/services/test_policy_service.py | 19 +- .../services/test_user_policy_mirroring.py | 238 +++++++++ .../unit/services/test_user_policy_service.py | 353 +++++++++++++- .../unit/test_migration_contract_artifacts.py | 7 +- tests/unit/test_migration_flags.py | 26 + tests/unit/test_query_parameters.py | 221 +++++++++ tests/unit/test_readiness.py | 46 ++ tests/unit/v2/test_alembic_v2.py | 63 ++- tests/unit/v2/test_database.py | 28 ++ tests/unit/v2/test_import_side_effects.py | 8 +- tests/unit/v2/test_metadata_routes.py | 12 +- tests/unit/v2/test_model_persistence.py | 246 +++++++++- tests/unit/v2/test_models.py | 126 ++++- tests/unit/v2/test_policy_canonicalization.py | 180 +++++++ tests/unit/v2/test_policy_catalog.py | 152 ++++++ tests/unit/v2/test_policy_commands.py | 146 ++++++ .../unit/v2/test_policy_legacy_translation.py | 201 ++++++++ .../v2/test_policy_migration_qualification.py | 131 +++++ .../v2/test_policy_persistence_statements.py | 26 + tests/unit/v2/test_policy_query.py | 188 ++++++++ tests/unit/v2/test_policy_routes.py | 421 ++++++++++++++++ tests/unit/v2/test_settings.py | 14 + tests/unit/v2/test_user_policy_legacy.py | 191 ++++++++ tests/unit/v2/test_user_policy_routes.py | 416 ++++++++++++++++ tests/unit/v2/test_user_policy_service.py | 213 ++++++++ uv.lock | 2 +- 98 files changed, 11681 insertions(+), 148 deletions(-) create mode 100644 docs/engineering/skills/api-routes.md create mode 100644 docs/migration/stage-10-v2-policies.md create mode 100644 migrations/v1/versions/3d6e8f553ca5_add_saved_policy_mirror_events.py create mode 100644 migrations/v2/versions/711ec2f0a5a5_migrate_v2_policies.py create mode 100644 migrations/v2/versions/c21c4a807a49_track_saved_policy_mirror_revisions.py create mode 100644 policyengine_api/data/v2/models/policy_mappings.py create mode 100644 policyengine_api/data/v2/policies/__init__.py create mode 100644 policyengine_api/data/v2/policies/api_schemas.py create mode 100644 policyengine_api/data/v2/policies/canonicalization.py create mode 100644 policyengine_api/data/v2/policies/catalog.py create mode 100644 policyengine_api/data/v2/policies/legacy.py create mode 100644 policyengine_api/data/v2/policies/persistence.py create mode 100644 policyengine_api/data/v2/policies/query.py create mode 100644 policyengine_api/data/v2/policies/schemas.py create mode 100644 policyengine_api/data/v2/policies/service.py create mode 100644 policyengine_api/data/v2/policy_migration_qualification.py create mode 100644 policyengine_api/data/v2/user_policies/__init__.py create mode 100644 policyengine_api/data/v2/user_policies/api_schemas.py create mode 100644 policyengine_api/data/v2/user_policies/legacy.py create mode 100644 policyengine_api/data/v2/user_policies/persistence.py create mode 100644 policyengine_api/data/v2/user_policies/query.py create mode 100644 policyengine_api/data/v2/user_policies/schemas.py create mode 100644 policyengine_api/data/v2/user_policies/service.py create mode 100644 policyengine_api/fastapi_routes/query_parameters.py create mode 100644 policyengine_api/fastapi_routes/v2_policies.py create mode 100644 policyengine_api/fastapi_routes/v2_user_policies.py create mode 100644 policyengine_api/query_parameters.py create mode 100644 policyengine_api/services/policy_mirroring.py create mode 100644 policyengine_api/services/user_policy_mirroring.py create mode 100644 scripts/qualify_v2_policy_migration.py create mode 100644 tests/contract/test_policy_v2_compatibility.py create mode 100644 tests/integration/test_v1_policy_dual_write.py create mode 100644 tests/integration/test_v1_user_policy_dual_write.py create mode 100644 tests/integration/test_v2_policy_persistence.py create mode 100644 tests/integration/test_v2_user_policy_mirroring.py create mode 100644 tests/unit/routes/test_policy_dual_write_routes.py create mode 100644 tests/unit/routes/test_user_policy_dual_write_routes.py create mode 100644 tests/unit/services/test_policy_mirroring.py create mode 100644 tests/unit/services/test_user_policy_mirroring.py create mode 100644 tests/unit/test_query_parameters.py create mode 100644 tests/unit/v2/test_policy_canonicalization.py create mode 100644 tests/unit/v2/test_policy_catalog.py create mode 100644 tests/unit/v2/test_policy_commands.py create mode 100644 tests/unit/v2/test_policy_legacy_translation.py create mode 100644 tests/unit/v2/test_policy_migration_qualification.py create mode 100644 tests/unit/v2/test_policy_persistence_statements.py create mode 100644 tests/unit/v2/test_policy_query.py create mode 100644 tests/unit/v2/test_policy_routes.py create mode 100644 tests/unit/v2/test_user_policy_legacy.py create mode 100644 tests/unit/v2/test_user_policy_routes.py create mode 100644 tests/unit/v2/test_user_policy_service.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index eb1e5e7ab..1dcd3de6b 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -3,6 +3,9 @@ Follow the repository's canonical engineering skills under `docs/engineering/skills/`. +For a new HTTP route or an existing route query-parameter contract change, +read `docs/engineering/skills/api-routes.md`. + For API v2 migration contract, route-group metadata, generated migration docs, or migration guard changes, read `docs/engineering/skills/migration_contracts.md`. diff --git a/AGENTS.md b/AGENTS.md index 3837d2f22..9007bf341 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,9 @@ Canonical AI-facing engineering skills live under `docs/engineering/skills/`. Use those files as the source of truth across Codex, Claude, Copilot, and other AI tools. +When adding an HTTP route or changing an existing route's query-parameter +contract, read `docs/engineering/skills/api-routes.md`. + When changing API v2 migration contracts, route-group migration metadata, PR cutover plans, generated migration docs, or migration guard scripts, read `docs/engineering/skills/migration_contracts.md`. diff --git a/CLAUDE.md b/CLAUDE.md index 6627f4b40..3e44f755f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,6 +13,9 @@ rules here. ## Required Skill Lookup +Before adding an HTTP route or changing an existing route's query-parameter +contract, read `docs/engineering/skills/api-routes.md`. + Before opening, replacing, or sharing a PR, read `docs/engineering/skills/github-prs.md`. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index 8cab684a5..7e615346f 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -7,8 +7,8 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | Metric | Count | | --- | ---: | | route group count | 9 | -| workflow count | 8 | -| request count | 32 | +| workflow count | 11 | +| request count | 43 | | db entity count | 6 | | sim flow count | 3 | @@ -19,7 +19,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check` | `none` | `none` | | `specification` | `specification` | `none` | `none` | | `metadata` | `metadata`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | -| `policy` | `policy`, `policies`, `user-policy` | `policy` | `none` | +| `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | | `household` | `household`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | | `simulation` | `simulation`, `simulations` | `simulation` | `economy` | @@ -39,6 +39,41 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/us/policy/{policy_id}` | 200 | `policy` | `status`, `message`, `result` | | `GET` | `/us/policies` | 200 | `policy` | `result` | +### `policy_resources_v2` + +- Current contract: `typed_v2_resources` +- Future owner: PR 10: Policy Migration + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/v2/policies?country_id=us` | 201 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at` | +| `GET` | `/v2/policies/{policy_id}?country_id=us` | 200 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at` | +| `GET` | `/v2/policies?country_id=us` | 200 | `policy` | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | + +### `saved_policy_v1_compatibility` + +- Current contract: `api_v1_compatible` +- Future owner: PR 10: Policy Migration + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/us/user-policy` | 201 | `policy` | `status`, `message`, `result.id`, `result.reform_id`, `result.reform_label`, `result.baseline_id`, `result.user_id` | +| `GET` | `/us/user-policy/{user_id}` | 200 | `policy` | `status`, `message`, `result` | +| `PUT` | `/us/user-policy` | 200 | `policy` | `status`, `message`, `result.id` | + +### `user_policy_associations_v2` + +- Current contract: `typed_v2_resources` +- Future owner: PR 10: Policy Migration + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/v2/user-policies?country_id=us` | 201 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.user_id`, `result.item.policy_id`, `result.item.name`, `result.item.description`, `result.item.created_at`, `result.item.updated_at` | +| `GET` | `/v2/user-policies/{association_id}?country_id=us` | 200 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.user_id`, `result.item.policy_id`, `result.item.name`, `result.item.description`, `result.item.created_at`, `result.item.updated_at` | +| `GET` | `/v2/user-policies?country_id=us&user_id=caller` | 200 | `policy` | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | +| `PATCH` | `/v2/user-policies/{association_id}?country_id=us` | 200 | `policy` | `status`, `message`, `result.item.id`, `result.item.name`, `result.item.description`, `result.item.updated_at` | +| `DELETE` | `/v2/user-policies/{association_id}?country_id=us` | 204 | `policy` | | + ### `household_save_edit_read` - Current contract: `api_v1_compatible` diff --git a/docs/engineering/skills/README.md b/docs/engineering/skills/README.md index a3248f5b2..a8d6708ed 100644 --- a/docs/engineering/skills/README.md +++ b/docs/engineering/skills/README.md @@ -9,6 +9,8 @@ first, then keep adapters thin. Current skills: +- `api-routes.md`: mandatory shared typed query-parameter conventions for new + routes and existing query-contract changes. - `alembic-migrations.md`: mandatory autogenerated Alembic revision workflow, deployment safeguards, and migration validation. - `github-prs.md`: PR workflow and migration PR handoff expectations. diff --git a/docs/engineering/skills/api-routes.md b/docs/engineering/skills/api-routes.md new file mode 100644 index 000000000..e0a293420 --- /dev/null +++ b/docs/engineering/skills/api-routes.md @@ -0,0 +1,105 @@ +# API Route Query Contracts + +Read this guidance before adding an HTTP route or changing the query-parameter +contract of an existing route. + +## Scope + +Every new route that accepts query parameters must define them through the +repository's shared typed query-parameter mechanism. When a change adds, +removes, renames, or changes the meaning of a query parameter on an existing +route, migrate that route's complete query contract to the shared mechanism as +part of the same change. + +Do not refactor an unrelated existing route merely because another part of its +module changes. This rule applies when the route is new or its public query +contract changes. + +## Canonical Parameter Meanings + +The framework-neutral source is `policyengine_api/query_parameters.py`. +Resource query models compose `CountryQuery`, `CatalogQuery`, +`PaginationQuery`, and the canonical annotated field types defined there. +FastAPI routes obtain dependencies through +`policyengine_api/fastapi_routes/query_parameters.py::query_dependency`. +Flask routes use `parse_multidict_query` when a reviewed legacy query contract +is migrated. Route modules must not reproduce these adapters. + +Reuse the shared definition whenever a query field has an established meaning. +The definition owns all of the following behavior: + +- public parameter name; +- Python and OpenAPI types; +- normalization and coercion; +- required or optional status; +- default value; +- length, numeric, enumeration, and collection bounds; +- scalar or list multiplicity; +- validation error semantics. + +Country, PolicyEngine.py version, pagination, UUID resource filters, and other +repeated filter concepts must not be redefined independently in route modules. +A resource-specific query schema should compose canonical fields and add only +filters whose meaning is specific to that resource. + +`country_id` is required only when the resource contract is country-scoped. Do +not add it to a route that has no country-dependent behavior merely for visual +uniformity. + +## Parsing Rules + +- Reject unknown query parameters rather than ignoring misspellings. +- Reject a scalar query parameter supplied more than once rather than selecting + an arbitrary value. +- Accept repeated keys only when the canonical field is explicitly list-valued. +- Keep query, path, and request-body fields in their documented locations; do + not move an identifier between them to reuse a schema. +- Do not maintain a second manual parser with different defaults or coercion. +- Do not copy parsing behavior from a legacy route when it conflicts with the + canonical typed definition. + +FastAPI routes should consume composed query schemas as typed dependencies so +the runtime validation and generated OpenAPI schema have the same source. A +Flask route whose query contract changes should use a thin adapter from +`request.args` into the same canonical schema rather than calling `get`, +`json.loads`, `int`, or similar coercion independently for each field. + +The Phase 10 policy schemas demonstrate the required composition: + +- `PolicyCreateQuery` for policy creation; +- `PolicyDetailQuery` for country-scoped policy detail; +- `PolicyCollectionQuery` for exact model filtering and pagination; +- `CountryQuery` for association create, detail, update, and delete; +- `UserPolicyCollectionQuery` for association user/policy filtering and + pagination. + +## Compatibility and Documentation + +Changing a query parameter's name, type, default, bounds, multiplicity, or +normalization changes the public route contract. Preserve existing behavior +unless the change explicitly authorizes a contract revision. For API v2 +migration routes, also read `migration_contracts.md` and update the migration +registry, workflow contracts, generated documentation, and stable OpenAPI +fields when applicable. + +The route's OpenAPI operation must expose every accepted query field with the +same required status, type, default, and bounds enforced at runtime. Do not +document query parameters accepted only by an untyped fallback parser. + +## Required Verification + +For each new or changed query contract, cover the applicable cases: + +- required parameters are absent; +- optional parameters use their canonical defaults; +- valid values receive canonical normalization; +- invalid types and out-of-range values are rejected; +- unknown parameters are rejected; +- duplicate scalar parameters are rejected; +- explicitly list-valued parameters accept the documented repeated form; +- OpenAPI declares the runtime name, type, required status, default, and bounds; +- parameters reused by multiple routes behave identically. + +Use `docs/engineering/skills/testing.md` to select the appropriate test layer +and commands. Route behavior that changes a migration contract also requires +the focused migration checks documented in `migration_contracts.md`. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 8af342cde..a8fb7e8f6 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -155,3 +155,63 @@ ruff check Commit only after formatting succeeds and changed Python files pass lint. If a broader repo-wide lint command fails on unrelated pre-existing issues, include that result in the handoff instead of hiding it. + +## Phase 10 Policy Migration + +Run the shared query, SQLModel, native policy and association, v1 compatibility, +configuration, readiness, and observability tests together: + +```bash +uv run pytest \ + tests/unit/test_query_parameters.py \ + tests/unit/v2/test_models.py \ + tests/unit/v2/test_model_persistence.py \ + tests/unit/v2/test_policy_routes.py \ + tests/unit/v2/test_user_policy_routes.py \ + tests/unit/v2/test_user_policy_service.py \ + tests/unit/services/test_policy_service.py \ + tests/unit/services/test_user_policy_service.py \ + tests/unit/services/test_policy_mirroring.py \ + tests/unit/services/test_user_policy_mirroring.py \ + tests/unit/routes/test_policy_dual_write_routes.py \ + tests/unit/routes/test_user_policy_dual_write_routes.py \ + tests/unit/test_migration_flags.py \ + tests/unit/test_readiness.py \ + tests/contract -q +``` + +Use only the reviewed disposable PostgreSQL target for the v2 lifecycle, +persistence, and cross-database transaction tests: + +```bash +V2_ALEMBIC_DISPOSABLE_TEST=1 \ +V2_MIGRATION_DATABASE_URL="postgresql+psycopg://.../policyengine_v2_alembic_test" \ +uv run pytest \ + tests/integration/test_alembic_v2_lifecycle.py \ + tests/integration/test_v2_policy_persistence.py \ + tests/integration/test_v1_policy_dual_write.py \ + tests/integration/test_v2_user_policy_mirroring.py \ + tests/integration/test_v1_user_policy_dual_write.py -q +``` + +Continue to run the isolated v1 MySQL lifecycle and compatibility suite because +Phase 10 adds source revision and mirror-event storage to Cloud SQL while it +must preserve every v1 read and response contract: + +```bash +uv run pytest \ + tests/integration/test_alembic_mysql_lifecycle.py \ + tests/contract/test_v1_route_contracts.py \ + tests/unit/services/test_policy_service.py \ + tests/unit/services/test_user_policy_service.py \ + tests/unit/routes/test_policy_dual_write_routes.py \ + tests/unit/routes/test_user_policy_dual_write_routes.py -q +``` + +Finally regenerate and validate migration contracts: + +```bash +python scripts/export_migration_contracts.py +python scripts/run_quality_guards.py +uv run pytest tests/contract tests/unit/test_migration_contract_artifacts.py -q +``` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index fa25f245c..8b1d026f5 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,10 +1,10 @@ { "metadata": { "db_entity_count": 6, - "request_count": 32, + "request_count": 43, "route_group_count": 9, "sim_flow_count": 3, - "workflow_count": 8 + "workflow_count": 11 }, "route_groups": [ { @@ -48,7 +48,8 @@ "path_segments": [ "policy", "policies", - "user-policy" + "user-policy", + "user-policies" ], "sim_flow": null }, @@ -136,6 +137,181 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "PR 10: Policy Migration", + "name": "policy_resources_v2", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/v2/policies?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/v2/policies/{policy_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/v2/policies?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.items", + "result.offset", + "result.limit", + "result.has_more" + ] + } + ] + }, + { + "current_contract": "api_v1_compatible", + "future_owner_pr": "PR 10: Policy Migration", + "name": "saved_policy_v1_compatibility", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/us/user-policy", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.id", + "result.reform_id", + "result.reform_label", + "result.baseline_id", + "result.user_id" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/us/user-policy/{user_id}", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result" + ] + }, + { + "expected_status": 200, + "method": "PUT", + "path": "/us/user-policy", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.id" + ] + } + ] + }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "PR 10: Policy Migration", + "name": "user_policy_associations_v2", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/v2/user-policies?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.user_id", + "result.item.policy_id", + "result.item.name", + "result.item.description", + "result.item.created_at", + "result.item.updated_at" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/v2/user-policies/{association_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.user_id", + "result.item.policy_id", + "result.item.name", + "result.item.description", + "result.item.created_at", + "result.item.updated_at" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/v2/user-policies?country_id=us&user_id=caller", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.items", + "result.offset", + "result.limit", + "result.has_more" + ] + }, + { + "expected_status": 200, + "method": "PATCH", + "path": "/v2/user-policies/{association_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.name", + "result.item.description", + "result.item.updated_at" + ] + }, + { + "expected_status": 204, + "method": "DELETE", + "path": "/v2/user-policies/{association_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [] + } + ] + }, { "current_contract": "api_v1_compatible", "future_owner_pr": "PR 11: Household Migration", diff --git a/docs/migration/stage-10-v2-policies.md b/docs/migration/stage-10-v2-policies.md new file mode 100644 index 000000000..43880a8f3 --- /dev/null +++ b/docs/migration/stage-10-v2-policies.md @@ -0,0 +1,127 @@ +# Stage 10 V2 Policy Deployment + +This runbook applies the reviewed v2 policy schema, activates native v2 policy +resources, and optionally requires immediate v1-to-v2 mirroring. V1 reads and +integer identifiers remain in Cloud SQL throughout this stage. + +## Preconditions + +1. Stage 9 metadata catalogs must be initialized for every supported country + and the running PolicyEngine.py version. Policy creation does not fall back + to v1 metadata or another catalog version. +2. The runtime Supabase URL and target identity settings must identify the same + reviewed project and require TLS. Use either the direct endpoint or + Supavisor session mode on port 5432. The runtime rejects transaction-pooling + endpoints on port 6543 because they cannot apply the per-session statement + timeout. +3. Before applying revision `711ec2f0a5a5`, run the dormant-table qualification + against the exact migration target: + + ```bash + V2_MIGRATION_DATABASE_URL="postgresql+psycopg://..." \ + V2_SUPABASE_PROJECT_REF="reviewed-project-reference" \ + V2_SUPABASE_ENVIRONMENT="staging" \ + python scripts/qualify_v2_policy_migration.py + ``` + + Continue only when the result reports zero policies, policy-owned parameter + values, and user-policy associations requiring preservation. A nonzero count + requires a separate preservation decision. + +## Schemas Before Traffic + +Apply both generated revisions before deploying code that can enable immediate +v1 saved-policy mirroring. The MySQL revision adds the source revision and +ordered event records; the PostgreSQL revision adds the destination's last +applied source revision. + +```bash +ALEMBIC_DATABASE_URL="mysql+pymysql://..." \ +uv run alembic -c alembic-v1.ini upgrade head +``` + +```bash +V2_MIGRATION_DATABASE_URL="postgresql+psycopg://..." \ +V2_SUPABASE_PROJECT_REF="reviewed-project-reference" \ +V2_SUPABASE_ENVIRONMENT="staging" \ +uv run alembic -c alembic-v2.ini upgrade head +``` + +Run each Alembic `check` command against the same corresponding target and +confirm no metadata/schema difference. The relevant generated revisions are +`3d6e8f553ca5` for MySQL and `c21c4a807a49` for PostgreSQL. + +## Activation + +Native `/v2/policies` and `/v2/user-policies` routes use only the server-side +Supabase connection. The routes are registered as preview resources; +`ROUTE_IMPL_POLICY=fastapi_native` declares them operational for deployment +readiness validation without moving v1 routes away from Flask. + +Keep the initial v1 settings explicit: + +```text +DB_READ_POLICY=cloud_sql +DB_WRITE_POLICY=cloud_sql +``` + +After native lifecycle checks pass, require immediate mirroring for v1 policy +and saved-policy mutations: + +```text +DB_READ_POLICY=cloud_sql +DB_WRITE_POLICY=dual_write +``` + +Under this selection, a core-policy mutation commits Cloud SQL first and then +completes its policy transaction in Supabase. A saved-policy mutation commits +its source row, incremented revision, and complete event in one Cloud SQL +transaction. The same request then processes that source's pending events in +revision order and records `processed_at` only after the corresponding +Supabase transaction commits. + +A Supabase failure returns HTTP 503. An identical client retry reads the +already committed v1 row, appends the next revision when applicable, and first +replays any retained earlier event. Destination revision and fingerprint +checks make a replay safe if Supabase committed but the Cloud SQL processing +marker did not. There is no background processor or reconciliation process. + +The v2 runtime applies one five-second value to pool acquisition, PostgreSQL +connection establishment, and each SQL statement. The API does not retry these +operations internally. Native v2 routes and v1 mirroring return a secret-safe +HTTP 503 for a timeout or other SQLAlchemy database failure so the caller can +retry the complete request. + +## Monitoring + +Monitor structured events with metric names `v1_policy_mirror_operations` and +`v1_user_policy_mirror_operations`. Alert on `outcome=error`, grouped by +`failure_category` and country. Events include the legacy integer ID, +destination UUID when committed, attempted and completed database sources, and +duration. They do not include policy JSON, presentation data, database URLs, or +credentials. + +Verify during staged activation that: + +- native policy and association create/read/list/update/delete operations use + the initialized catalog and Supabase only; +- both newly created and existing v1 policies receive durable mappings; +- v1 saved-policy label updates change association `name`, while v1-only field + updates change only the mapping fingerprint; +- saved-policy event revisions are processed in ascending order, successful + events have `processed_at`, and failed events retain a null `processed_at`; +- all v1 reads continue to query Cloud SQL and all v1 responses retain integer + IDs without v2 UUIDs. + +## Application Rollback + +Set `DB_WRITE_POLICY=cloud_sql` or route traffic to the prior application +revision. This immediately removes the Supabase requirement from v1 mutations. +Keep `DB_READ_POLICY=cloud_sql`. Do not delete policies, associations, or +mappings already committed in Supabase; they remain valid for later retries. + +Application rollback does not automatically downgrade either additive source +schema or the v2 schema. If a schema downgrade is separately approved, first +disable native policy traffic and mirroring, verify that no pending Cloud SQL +events or retained v2 data depend on the revisions, and run the reviewed +Alembic downgrades against their confirmed database targets. diff --git a/migrations/v1/versions/3d6e8f553ca5_add_saved_policy_mirror_events.py b/migrations/v1/versions/3d6e8f553ca5_add_saved_policy_mirror_events.py new file mode 100644 index 000000000..d393680c9 --- /dev/null +++ b/migrations/v1/versions/3d6e8f553ca5_add_saved_policy_mirror_events.py @@ -0,0 +1,103 @@ +"""add saved policy mirror events + +Revision ID: 3d6e8f553ca5 +Revises: 1914c0422236 +Create Date: 2026-09-01 19:35:29.593649 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "3d6e8f553ca5" +down_revision: Union[str, None] = "1914c0422236" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "user_policy_mirror_events", + sa.Column( + "id", + sa.BigInteger().with_variant(sa.Integer(), "sqlite"), + autoincrement=True, + nullable=False, + ), + sa.Column("country_id", sa.String(length=3), nullable=False), + sa.Column("legacy_user_policy_id", sa.Integer(), nullable=False), + sa.Column("source_revision", sa.BigInteger(), nullable=False), + sa.Column("event_type", sa.String(length=16), nullable=False), + sa.Column("payload_schema_version", sa.SmallInteger(), nullable=False), + sa.Column("payload_json", sa.JSON(), nullable=False), + sa.Column("source_fingerprint_sha256", sa.String(length=64), nullable=False), + sa.Column( + "created_at", + sa.DateTime(), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column("processed_at", sa.DateTime(), nullable=True), + sa.CheckConstraint( + "event_type IN ('create', 'update')", + name="ck_user_policy_mirror_events_event_type", + ), + sa.CheckConstraint( + "length(source_fingerprint_sha256) = 64", + name="ck_user_policy_mirror_events_fingerprint_length", + ), + sa.CheckConstraint( + "payload_schema_version > 0", + name="ck_user_policy_mirror_events_payload_schema_version", + ), + sa.CheckConstraint( + "source_revision > 0", name="ck_user_policy_mirror_events_source_revision" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "country_id", + "legacy_user_policy_id", + "source_revision", + name="uq_user_policy_mirror_events_source_revision", + ), + ) + op.create_index( + "ix_user_policy_mirror_events_pending_age", + "user_policy_mirror_events", + ["processed_at", "created_at"], + unique=False, + ) + op.create_index( + "ix_user_policy_mirror_events_pending_source", + "user_policy_mirror_events", + ["country_id", "legacy_user_policy_id", "processed_at", "source_revision"], + unique=False, + ) + op.add_column( + "user_policies", + sa.Column( + "mirror_revision", + sa.BigInteger(), + server_default=sa.text("0"), + nullable=False, + ), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("user_policies", "mirror_revision") + op.drop_index( + "ix_user_policy_mirror_events_pending_source", + table_name="user_policy_mirror_events", + ) + op.drop_index( + "ix_user_policy_mirror_events_pending_age", + table_name="user_policy_mirror_events", + ) + op.drop_table("user_policy_mirror_events") + # ### end Alembic commands ### diff --git a/migrations/v2/versions/711ec2f0a5a5_migrate_v2_policies.py b/migrations/v2/versions/711ec2f0a5a5_migrate_v2_policies.py new file mode 100644 index 000000000..553eb73f7 --- /dev/null +++ b/migrations/v2/versions/711ec2f0a5a5_migrate_v2_policies.py @@ -0,0 +1,395 @@ +"""migrate v2 policies + +Revision ID: 711ec2f0a5a5 +Revises: 68b4a5ae5dc5 +Create Date: 2026-09-01 16:17:04.855675 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import postgresql + +revision: str = "711ec2f0a5a5" +down_revision: Union[str, None] = "68b4a5ae5dc5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "parameter_values", + "value_json", + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant( + postgresql.JSONB(astext_type=sa.Text()), "postgresql" + ), + existing_nullable=False, + ) + op.create_unique_constraint( + "uq_parameter_values_policy_parameter_start_date", + "parameter_values", + ["policy_id", "parameter_id", "start_date"], + ) + op.create_check_constraint( + op.f("ck_parameter_values_effective_period"), + "parameter_values", + "end_date IS NULL OR end_date >= start_date", + ) + op.add_column( + "policies", + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + ) + op.add_column( + "policies", sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False) + ) + op.add_column( + "policies", sa.Column("canonicalization_version", sa.Integer(), nullable=False) + ) + op.add_column( + "policies", + sa.Column( + "content_hash", sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False + ), + ) + op.create_index( + "ix_policies_country_model", + "policies", + ["country_id", "tax_benefit_model_id"], + unique=False, + ) + op.create_index( + "ix_policies_country_model_version", + "policies", + ["country_id", "tax_benefit_model_version_id"], + unique=False, + ) + op.create_index( + op.f("ix_policies_tax_benefit_model_version_id"), + "policies", + ["tax_benefit_model_version_id"], + unique=False, + ) + op.create_unique_constraint( + "uq_policies_canonicalization_content_hash", + "policies", + ["canonicalization_version", "content_hash"], + ) + op.create_unique_constraint( + "uq_policies_id_country", "policies", ["id", "country_id"] + ) + op.create_foreign_key( + op.f("fk_policies_tax_benefit_model_version_id_tax_benefit_model_versions"), + "policies", + "tax_benefit_model_versions", + ["tax_benefit_model_version_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_check_constraint( + op.f("ck_policies_canonicalization_version"), + "policies", + "canonicalization_version > 0", + ) + op.create_check_constraint( + op.f("ck_policies_content_hash_length"), "policies", "length(content_hash) = 64" + ) + op.create_check_constraint( + op.f("ck_policies_country"), "policies", "country_id IN ('us', 'uk')" + ) + op.drop_column("policies", "name") + op.drop_column("policies", "description") + # Post-generation correction: the generated mapping table preceded the + # composite policy key referenced by its foreign key. + op.create_table( + "legacy_policy_mappings", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + sa.Column("legacy_policy_id", sa.BigInteger(), nullable=False), + sa.Column("policy_id", sa.Uuid(), nullable=False), + sa.Column( + "source_policy_hash", + sqlmodel.sql.sqltypes.AutoString(length=255), + nullable=False, + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", name=op.f("ck_legacy_policy_mappings_country") + ), + sa.ForeignKeyConstraint( + ["policy_id", "country_id"], + ["policies.id", "policies.country_id"], + name="fk_legacy_policy_mappings_policy_country", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_legacy_policy_mappings")), + sa.UniqueConstraint( + "country_id", + "legacy_policy_id", + name="uq_legacy_policy_mappings_country_legacy", + ), + ) + op.create_index( + "ix_legacy_policy_mappings_policy", + "legacy_policy_mappings", + ["policy_id"], + unique=False, + ) + op.add_column( + "user_policies", + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + ) + op.add_column( + "user_policies", + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + ) + op.add_column("user_policies", sa.Column("description", sa.Text(), nullable=True)) + # Post-generation correction: PostgreSQL cannot change the source type + # while its UUID foreign key remains in place. + op.drop_constraint( + op.f("fk_user_policies_user_id_users"), "user_policies", type_="foreignkey" + ) + op.alter_column( + "user_policies", + "user_id", + existing_type=sa.UUID(), + type_=sqlmodel.sql.sqltypes.AutoString(length=255), + existing_nullable=False, + postgresql_using="user_id::text", + ) + op.drop_constraint( + op.f("uq_user_policies_user_policy"), "user_policies", type_="unique" + ) + op.create_index( + "ix_user_policies_country_policy", + "user_policies", + ["country_id", "policy_id"], + unique=False, + ) + op.create_index( + "ix_user_policies_country_user_created_id", + "user_policies", + ["country_id", "user_id", "created_at", "id"], + unique=False, + ) + op.create_unique_constraint( + "uq_user_policies_id_country", "user_policies", ["id", "country_id"] + ) + op.drop_constraint( + op.f("fk_user_policies_policy_id_policies"), "user_policies", type_="foreignkey" + ) + op.create_foreign_key( + "fk_user_policies_policy_country", + "user_policies", + "policies", + ["policy_id", "country_id"], + ["id", "country_id"], + ondelete="RESTRICT", + ) + op.create_check_constraint( + op.f("ck_user_policies_country"), "user_policies", "country_id IN ('us', 'uk')" + ) + op.drop_column("user_policies", "country") + op.drop_column("user_policies", "label") + # Post-generation correction: this mapping likewise depends on the new + # association composite key. + op.create_table( + "legacy_user_policy_mappings", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + sa.Column("legacy_user_policy_id", sa.BigInteger(), nullable=False), + sa.Column("user_policy_id", sa.Uuid(), nullable=False), + sa.Column("fingerprint_version", sa.Integer(), nullable=False), + sa.Column( + "fingerprint_sha256", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=False, + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name=op.f("ck_legacy_user_policy_mappings_country"), + ), + sa.CheckConstraint( + "fingerprint_version > 0", + name=op.f("ck_legacy_user_policy_mappings_fingerprint_version"), + ), + sa.CheckConstraint( + "length(fingerprint_sha256) = 64", + name=op.f("ck_legacy_user_policy_mappings_fingerprint_length"), + ), + sa.ForeignKeyConstraint( + ["user_policy_id", "country_id"], + ["user_policies.id", "user_policies.country_id"], + name="fk_legacy_user_policy_mappings_association_country", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_legacy_user_policy_mappings")), + sa.UniqueConstraint( + "country_id", + "legacy_user_policy_id", + name="uq_legacy_user_policy_mappings_country_legacy", + ), + sa.UniqueConstraint( + "user_policy_id", name="uq_legacy_user_policy_mappings_association" + ), + ) + op.create_index( + "ix_legacy_user_policy_mappings_association", + "legacy_user_policy_mappings", + ["user_policy_id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # Post-generation correction: dependent mappings must be removed before + # their referenced composite keys. + op.drop_index( + "ix_legacy_user_policy_mappings_association", + table_name="legacy_user_policy_mappings", + ) + op.drop_table("legacy_user_policy_mappings") + op.drop_index( + "ix_legacy_policy_mappings_policy", table_name="legacy_policy_mappings" + ) + op.drop_table("legacy_policy_mappings") + op.add_column( + "user_policies", + sa.Column("label", sa.VARCHAR(length=255), autoincrement=False, nullable=True), + ) + op.add_column( + "user_policies", + sa.Column( + "country", sa.VARCHAR(length=16), autoincrement=False, nullable=False + ), + ) + op.drop_constraint(op.f("ck_user_policies_country"), "user_policies", type_="check") + op.drop_constraint( + "fk_user_policies_policy_country", "user_policies", type_="foreignkey" + ) + op.create_foreign_key( + op.f("fk_user_policies_policy_id_policies"), + "user_policies", + "policies", + ["policy_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_constraint("uq_user_policies_id_country", "user_policies", type_="unique") + op.drop_index( + "ix_user_policies_country_user_created_id", table_name="user_policies" + ) + op.drop_index("ix_user_policies_country_policy", table_name="user_policies") + op.create_unique_constraint( + op.f("uq_user_policies_user_policy"), + "user_policies", + ["user_id", "policy_id"], + postgresql_nulls_not_distinct=False, + ) + op.alter_column( + "user_policies", + "user_id", + existing_type=sqlmodel.sql.sqltypes.AutoString(length=255), + type_=sa.UUID(), + existing_nullable=False, + postgresql_using="user_id::uuid", + ) + op.create_foreign_key( + op.f("fk_user_policies_user_id_users"), + "user_policies", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("user_policies", "description") + op.drop_column("user_policies", "name") + op.drop_column("user_policies", "country_id") + op.add_column( + "policies", + sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), + ) + op.add_column( + "policies", + sa.Column("name", sa.VARCHAR(length=255), autoincrement=False, nullable=False), + ) + op.drop_constraint(op.f("ck_policies_country"), "policies", type_="check") + op.drop_constraint( + op.f("ck_policies_content_hash_length"), "policies", type_="check" + ) + op.drop_constraint( + op.f("ck_policies_canonicalization_version"), "policies", type_="check" + ) + op.drop_constraint( + op.f("fk_policies_tax_benefit_model_version_id_tax_benefit_model_versions"), + "policies", + type_="foreignkey", + ) + op.drop_constraint("uq_policies_id_country", "policies", type_="unique") + op.drop_constraint( + "uq_policies_canonicalization_content_hash", "policies", type_="unique" + ) + op.drop_index( + op.f("ix_policies_tax_benefit_model_version_id"), table_name="policies" + ) + op.drop_index("ix_policies_country_model_version", table_name="policies") + op.drop_index("ix_policies_country_model", table_name="policies") + op.drop_column("policies", "content_hash") + op.drop_column("policies", "canonicalization_version") + op.drop_column("policies", "tax_benefit_model_version_id") + op.drop_column("policies", "country_id") + op.drop_constraint( + op.f("ck_parameter_values_effective_period"), "parameter_values", type_="check" + ) + op.drop_constraint( + "uq_parameter_values_policy_parameter_start_date", + "parameter_values", + type_="unique", + ) + op.alter_column( + "parameter_values", + "value_json", + existing_type=sa.JSON().with_variant( + postgresql.JSONB(astext_type=sa.Text()), "postgresql" + ), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=False, + ) + # ### end Alembic commands ### diff --git a/migrations/v2/versions/c21c4a807a49_track_saved_policy_mirror_revisions.py b/migrations/v2/versions/c21c4a807a49_track_saved_policy_mirror_revisions.py new file mode 100644 index 000000000..91042f206 --- /dev/null +++ b/migrations/v2/versions/c21c4a807a49_track_saved_policy_mirror_revisions.py @@ -0,0 +1,49 @@ +"""track saved policy mirror revisions + +Revision ID: c21c4a807a49 +Revises: 711ec2f0a5a5 +Create Date: 2026-09-01 19:35:37.381580 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "c21c4a807a49" +down_revision: Union[str, None] = "711ec2f0a5a5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "legacy_user_policy_mappings", + sa.Column( + "last_applied_source_revision", + sa.BigInteger(), + server_default="0", + nullable=False, + ), + ) + op.create_check_constraint( + op.f("ck_legacy_user_policy_mappings_source_revision"), + "legacy_user_policy_mappings", + "last_applied_source_revision >= 0", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("ck_legacy_user_policy_mappings_source_revision"), + "legacy_user_policy_mappings", + type_="check", + ) + op.drop_column("legacy_user_policy_mappings", "last_applied_source_revision") + # ### end Alembic commands ### diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index 171cf0c61..933d0a62b 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -19,7 +19,14 @@ from policyengine_api.fastapi_routes.specification import ( build_specification_router, ) +from policyengine_api.fastapi_routes.v2_policies import ( + PolicyRequestTooLargeError, + policy_error_response, +) from policyengine_api.fastapi_routes.v2_metadata import build_v2_metadata_router +from policyengine_api.fastapi_routes.v2_user_policies import ( + user_policy_error_response, +) from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, @@ -113,6 +120,13 @@ async def typed_v2_request_validation_error( error: RequestValidationError, ) -> Response: if request.url.path.startswith("/v2/"): + if request.url.path.startswith("/v2/user-policies"): + return user_policy_error_response( + 422, + "Invalid v2 user-policy request", + ) + if request.url.path.startswith("/v2/policies"): + return policy_error_response(422, "Invalid v2 policy request") from policyengine_api.fastapi_routes.v2_metadata_common import ( error_response, ) @@ -120,6 +134,13 @@ async def typed_v2_request_validation_error( return error_response(422, "Invalid v2 metadata request") return await request_validation_exception_handler(request, error) + @app.exception_handler(PolicyRequestTooLargeError) + async def oversized_policy_request( + _request: Request, + error: PolicyRequestTooLargeError, + ) -> Response: + return policy_error_response(413, str(error)) + @app.middleware("http") async def add_cors_for_native_routes(request, call_next): started_at = time.time() diff --git a/policyengine_api/data/v1_models.py b/policyengine_api/data/v1_models.py index 6e848d106..df3b58362 100644 --- a/policyengine_api/data/v1_models.py +++ b/policyengine_api/data/v1_models.py @@ -12,9 +12,12 @@ from sqlalchemy import ( BigInteger, CHAR, + CheckConstraint, DateTime, + Index, Integer, JSON, + SmallInteger, String, Text, UniqueConstraint, @@ -127,6 +130,73 @@ class UserPolicy(V1Base): updated_date: Mapped[int] = mapped_column(BigInteger) budgetary_impact: Mapped[str | None] = mapped_column(String(255)) type: Mapped[str | None] = mapped_column(String(255)) + mirror_revision: Mapped[int] = mapped_column( + BigInteger, + nullable=False, + default=0, + server_default=text("0"), + ) + + +class UserPolicyMirrorEvent(V1Base): + """Ordered, durable input for synchronous saved-policy mirroring.""" + + __tablename__ = "user_policy_mirror_events" + __table_args__ = ( + UniqueConstraint( + "country_id", + "legacy_user_policy_id", + "source_revision", + name="uq_user_policy_mirror_events_source_revision", + ), + CheckConstraint( + "source_revision > 0", + name="ck_user_policy_mirror_events_source_revision", + ), + CheckConstraint( + "event_type IN ('create', 'update')", + name="ck_user_policy_mirror_events_event_type", + ), + CheckConstraint( + "payload_schema_version > 0", + name="ck_user_policy_mirror_events_payload_schema_version", + ), + CheckConstraint( + "length(source_fingerprint_sha256) = 64", + name="ck_user_policy_mirror_events_fingerprint_length", + ), + Index( + "ix_user_policy_mirror_events_pending_source", + "country_id", + "legacy_user_policy_id", + "processed_at", + "source_revision", + ), + Index( + "ix_user_policy_mirror_events_pending_age", + "processed_at", + "created_at", + ), + ) + + id: Mapped[int] = mapped_column( + BigInteger().with_variant(Integer, "sqlite"), + primary_key=True, + autoincrement=True, + ) + country_id: Mapped[str] = mapped_column(String(3)) + legacy_user_policy_id: Mapped[int] = mapped_column(Integer) + source_revision: Mapped[int] = mapped_column(BigInteger) + event_type: Mapped[str] = mapped_column(String(16)) + payload_schema_version: Mapped[int] = mapped_column(SmallInteger) + payload_json: Mapped[Any] = mapped_column(JSON) + source_fingerprint_sha256: Mapped[str] = mapped_column(String(64)) + created_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=text("CURRENT_TIMESTAMP"), + ) + processed_at: Mapped[datetime | None] = mapped_column(DateTime) class UserProfile(V1Base): diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 3e59306cf..9b028cd92 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -32,7 +32,7 @@ ) -EXPECTED_ALEMBIC_REVISION = "68b4a5ae5dc5" +EXPECTED_ALEMBIC_REVISION = "c21c4a807a49" # Stable application-defined PostgreSQL lock ID shared by all v2 catalog publishers. PUBLICATION_ADVISORY_LOCK_KEY = 8_629_020_026_090_001 diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py index d9922ac06..97215a526 100644 --- a/policyengine_api/data/v2/database.py +++ b/policyengine_api/data/v2/database.py @@ -20,7 +20,7 @@ DATABASE_POOL_RECYCLE_SECONDS = 1800 DATABASE_POOL_SIZE = 5 DATABASE_POOL_MAX_OVERFLOW = 5 -DATABASE_POOL_TIMEOUT_SECONDS = 30 +DATABASE_TIMEOUT_SECONDS = 5 _state_lock = Lock() _engine: Engine | None = None @@ -55,7 +55,11 @@ def build_v2_engine(settings: V2DatabaseSettings) -> Engine: pool_recycle=DATABASE_POOL_RECYCLE_SECONDS, pool_size=DATABASE_POOL_SIZE, max_overflow=DATABASE_POOL_MAX_OVERFLOW, - pool_timeout=DATABASE_POOL_TIMEOUT_SECONDS, + pool_timeout=DATABASE_TIMEOUT_SECONDS, + connect_args={ + "connect_timeout": DATABASE_TIMEOUT_SECONDS, + "options": (f"-c statement_timeout={DATABASE_TIMEOUT_SECONDS * 1_000}"), + }, ) diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 491207ef4..253073562 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -51,6 +51,10 @@ UserReportAssociation, UserSimulationAssociation, ) +from policyengine_api.data.v2.models.policy_mappings import ( # noqa: E402 + LegacyPolicyMapping, + LegacyUserPolicyMapping, +) from policyengine_api.data.v2.models.reports import ( # noqa: E402 AggregateOutput, AggregateType, @@ -94,6 +98,8 @@ "Inequality", "IntraDecileImpact", "LocalAuthorityImpact", + "LegacyPolicyMapping", + "LegacyUserPolicyMapping", "OutputStatus", "Parameter", "ParameterNode", diff --git a/policyengine_api/data/v2/models/associations.py b/policyengine_api/data/v2/models/associations.py index e15dd60e6..0581defe0 100644 --- a/policyengine_api/data/v2/models/associations.py +++ b/policyengine_api/data/v2/models/associations.py @@ -1,7 +1,7 @@ """Canonical SQLModel tables linking v2 users to their domain records.""" from datetime import datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from uuid import UUID import sqlalchemy as sa @@ -14,6 +14,9 @@ from policyengine_api.data.v2.models.users import User if TYPE_CHECKING: + from policyengine_api.data.v2.models.policy_mappings import ( + LegacyUserPolicyMapping, + ) from policyengine_api.data.v2.models.reports import Report @@ -48,27 +51,45 @@ class UserPolicy(TimestampedModel, table=True): __tablename__ = "user_policies" __table_args__ = ( sa.UniqueConstraint( + "id", + "country_id", + name="uq_user_policies_id_country", + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name="ck_user_policies_country", + ), + sa.ForeignKeyConstraint( + ["policy_id", "country_id"], + ["policies.id", "policies.country_id"], + name="fk_user_policies_policy_country", + ondelete="RESTRICT", + ), + sa.Index( + "ix_user_policies_country_user_created_id", + "country_id", "user_id", + "created_at", + "id", + ), + sa.Index( + "ix_user_policies_country_policy", + "country_id", "policy_id", - name="uq_user_policies_user_policy", ), ) - user_id: UUID = Field( - foreign_key="users.id", - ondelete="CASCADE", - index=True, - ) - policy_id: UUID = Field( - foreign_key="policies.id", - ondelete="CASCADE", - index=True, - ) - country: str = Field(max_length=16) - label: str | None = Field(default=None, max_length=255) + user_id: str = Field(max_length=255, index=True) + policy_id: UUID = Field(index=True) + country_id: str = Field(max_length=2) + name: str | None = Field(default=None, max_length=255) + description: str | None = Field(default=None, sa_type=sa.Text) - user: User = Relationship(back_populates="policy_associations") policy: Policy = Relationship(back_populates="user_associations") + legacy_mapping: Optional["LegacyUserPolicyMapping"] = Relationship( + back_populates="association", + cascade_delete=True, + ) class UserSimulationAssociation(TimestampedModel, table=True): diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index ab452d5e2..d1a797187 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -87,6 +87,7 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): ) datasets: list["Dataset"] = Relationship(back_populates="tax_benefit_model_version") regions: list["Region"] = Relationship(back_populates="tax_benefit_model_version") + policies: list["Policy"] = Relationship(back_populates="tax_benefit_model_version") class Region(TimestampedModel, table=True): @@ -314,6 +315,16 @@ class ParameterValue(IdentifiedModel, table=True): "policy_id IS NULL OR dynamic_id IS NULL", name="ck_parameter_values_single_owner", ), + sa.CheckConstraint( + "end_date IS NULL OR end_date >= start_date", + name="ck_parameter_values_effective_period", + ), + sa.UniqueConstraint( + "policy_id", + "parameter_id", + "start_date", + name="uq_parameter_values_policy_parameter_start_date", + ), sa.Index( "ix_parameter_values_parameter_period", "parameter_id", @@ -337,7 +348,9 @@ class ParameterValue(IdentifiedModel, table=True): foreign_key="parameters.id", ondelete="CASCADE", ) - value_json: Any = Field(sa_type=sa.JSON) + value_json: Any = Field( + sa_type=sa.JSON().with_variant(sa.dialects.postgresql.JSONB(), "postgresql") + ) start_date: datetime = Field(sa_type=sa.DateTime(timezone=True)) end_date: datetime | None = Field( default=None, diff --git a/policyengine_api/data/v2/models/policies.py b/policyengine_api/data/v2/models/policies.py index ba632a240..9b9957b30 100644 --- a/policyengine_api/data/v2/models/policies.py +++ b/policyengine_api/data/v2/models/policies.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING from uuid import UUID +import sqlalchemy as sa from sqlmodel import Field, Relationship from policyengine_api.data.v2.models.base import TimestampedModel @@ -11,23 +12,70 @@ if TYPE_CHECKING: from policyengine_api.data.v2.models.associations import UserPolicy from policyengine_api.data.v2.models.households import HouseholdJob - from policyengine_api.data.v2.models.metadata import ParameterValue + from policyengine_api.data.v2.models.metadata import ( + ParameterValue, + TaxBenefitModelVersion, + ) + from policyengine_api.data.v2.models.policy_mappings import LegacyPolicyMapping from policyengine_api.data.v2.models.reports import Report from policyengine_api.data.v2.models.simulations import Simulation class Policy(TimestampedModel, table=True): __tablename__ = "policies" + __table_args__ = ( + sa.UniqueConstraint( + "id", + "country_id", + name="uq_policies_id_country", + ), + sa.UniqueConstraint( + "canonicalization_version", + "content_hash", + name="uq_policies_canonicalization_content_hash", + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name="ck_policies_country", + ), + sa.CheckConstraint( + "canonicalization_version > 0", + name="ck_policies_canonicalization_version", + ), + sa.CheckConstraint( + "length(content_hash) = 64", + name="ck_policies_content_hash_length", + ), + sa.Index( + "ix_policies_country_model", + "country_id", + "tax_benefit_model_id", + ), + sa.Index( + "ix_policies_country_model_version", + "country_id", + "tax_benefit_model_version_id", + ), + ) - name: str = Field(max_length=255) - description: str | None = None + country_id: str = Field(max_length=2) tax_benefit_model_id: UUID = Field( foreign_key="tax_benefit_models.id", ondelete="RESTRICT", index=True, ) + tax_benefit_model_version_id: UUID = Field( + foreign_key="tax_benefit_model_versions.id", + ondelete="RESTRICT", + index=True, + ) + canonicalization_version: int + content_hash: str = Field(max_length=64) tax_benefit_model: TaxBenefitModel = Relationship(back_populates="policies") + tax_benefit_model_version: "TaxBenefitModelVersion" = Relationship( + back_populates="policies" + ) parameter_values: list["ParameterValue"] = Relationship( back_populates="policy", cascade_delete=True, @@ -37,8 +85,8 @@ class Policy(TimestampedModel, table=True): reports: list["Report"] = Relationship(back_populates="policy") user_associations: list["UserPolicy"] = Relationship( back_populates="policy", - cascade_delete=True, ) + legacy_mappings: list["LegacyPolicyMapping"] = Relationship(back_populates="policy") class Dynamic(TimestampedModel, table=True): diff --git a/policyengine_api/data/v2/models/policy_mappings.py b/policyengine_api/data/v2/models/policy_mappings.py new file mode 100644 index 000000000..b08d04b8c --- /dev/null +++ b/policyengine_api/data/v2/models/policy_mappings.py @@ -0,0 +1,103 @@ +"""Durable source-identity mappings for immediate v1 policy mirroring.""" + +from typing import TYPE_CHECKING +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Field, Relationship + +from policyengine_api.data.v2.models.base import TimestampedModel + +if TYPE_CHECKING: + from policyengine_api.data.v2.models.associations import UserPolicy + from policyengine_api.data.v2.models.policies import Policy + + +class LegacyPolicyMapping(TimestampedModel, table=True): + """Map one country-scoped v1 policy ID to deduplicated v2 content.""" + + __tablename__ = "legacy_policy_mappings" + __table_args__ = ( + sa.UniqueConstraint( + "country_id", + "legacy_policy_id", + name="uq_legacy_policy_mappings_country_legacy", + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name="ck_legacy_policy_mappings_country", + ), + sa.ForeignKeyConstraint( + ["policy_id", "country_id"], + ["policies.id", "policies.country_id"], + name="fk_legacy_policy_mappings_policy_country", + ondelete="RESTRICT", + ), + sa.Index( + "ix_legacy_policy_mappings_policy", + "policy_id", + ), + ) + + country_id: str = Field(max_length=2) + legacy_policy_id: int = Field(sa_type=sa.BigInteger) + policy_id: UUID + source_policy_hash: str = Field(max_length=255) + + policy: "Policy" = Relationship(back_populates="legacy_mappings") + + +class LegacyUserPolicyMapping(TimestampedModel, table=True): + """Map one country-scoped v1 saved policy to one v2 association.""" + + __tablename__ = "legacy_user_policy_mappings" + __table_args__ = ( + sa.UniqueConstraint( + "country_id", + "legacy_user_policy_id", + name="uq_legacy_user_policy_mappings_country_legacy", + ), + sa.UniqueConstraint( + "user_policy_id", + name="uq_legacy_user_policy_mappings_association", + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name="ck_legacy_user_policy_mappings_country", + ), + sa.CheckConstraint( + "fingerprint_version > 0", + name="ck_legacy_user_policy_mappings_fingerprint_version", + ), + sa.CheckConstraint( + "last_applied_source_revision >= 0", + name="ck_legacy_user_policy_mappings_source_revision", + ), + sa.CheckConstraint( + "length(fingerprint_sha256) = 64", + name="ck_legacy_user_policy_mappings_fingerprint_length", + ), + sa.ForeignKeyConstraint( + ["user_policy_id", "country_id"], + ["user_policies.id", "user_policies.country_id"], + name="fk_legacy_user_policy_mappings_association_country", + ondelete="CASCADE", + ), + sa.Index( + "ix_legacy_user_policy_mappings_association", + "user_policy_id", + ), + ) + + country_id: str = Field(max_length=2) + legacy_user_policy_id: int = Field(sa_type=sa.BigInteger) + user_policy_id: UUID + last_applied_source_revision: int = Field( + default=0, + sa_type=sa.BigInteger, + sa_column_kwargs={"server_default": "0"}, + ) + fingerprint_version: int + fingerprint_sha256: str = Field(max_length=64) + + association: "UserPolicy" = Relationship(back_populates="legacy_mapping") diff --git a/policyengine_api/data/v2/models/users.py b/policyengine_api/data/v2/models/users.py index fc7582a5a..6a5a62ae9 100644 --- a/policyengine_api/data/v2/models/users.py +++ b/policyengine_api/data/v2/models/users.py @@ -10,7 +10,6 @@ if TYPE_CHECKING: from policyengine_api.data.v2.models.associations import ( UserHouseholdAssociation, - UserPolicy, UserReportAssociation, UserSimulationAssociation, ) @@ -37,10 +36,6 @@ class User(IdentifiedModel, table=True): back_populates="user", cascade_delete=True, ) - policy_associations: list["UserPolicy"] = Relationship( - back_populates="user", - cascade_delete=True, - ) simulation_associations: list["UserSimulationAssociation"] = Relationship( back_populates="user", cascade_delete=True, diff --git a/policyengine_api/data/v2/policies/__init__.py b/policyengine_api/data/v2/policies/__init__.py new file mode 100644 index 000000000..abc6de984 --- /dev/null +++ b/policyengine_api/data/v2/policies/__init__.py @@ -0,0 +1 @@ +"""Immutable v2 policy validation, persistence, and read operations.""" diff --git a/policyengine_api/data/v2/policies/api_schemas.py b/policyengine_api/data/v2/policies/api_schemas.py new file mode 100644 index 000000000..bbe5d64df --- /dev/null +++ b/policyengine_api/data/v2/policies/api_schemas.py @@ -0,0 +1,125 @@ +"""Strict HTTP schemas for the native v2 policy API.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Annotated, Generic, Literal, TypeVar +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, JsonValue, StringConstraints + +from policyengine_api.data.v2.policies.query import PolicyPage, PolicyRead +from policyengine_api.data.v2.policies.schemas import PolicyCreateCommand + + +MAXIMUM_POLICY_REQUEST_BYTES = 1_048_576 + + +class StrictPolicyAPIModel(BaseModel): + """Strict request/response base with dataclass conversion support.""" + + model_config = ConfigDict( + extra="forbid", + allow_inf_nan=False, + from_attributes=True, + ) + + +class PolicyCreateRequest(PolicyCreateCommand): + """Native body containing immutable policy content only.""" + + +class PolicyParameterValueItem(StrictPolicyAPIModel): + id: UUID + parameter_id: UUID + parameter_name: str + value: JsonValue + start_date: datetime + end_date: datetime | None + + +class PolicyItem(StrictPolicyAPIModel): + id: UUID + country_id: str + tax_benefit_model_id: UUID + created_at: datetime + updated_at: datetime + parameter_values: list[PolicyParameterValueItem] + + @classmethod + def from_read(cls, item: PolicyRead) -> "PolicyItem": + return cls.model_validate(item) + + +class PolicyDetailResult(StrictPolicyAPIModel): + item: PolicyItem + + +class PolicyPageResult(StrictPolicyAPIModel): + items: list[PolicyItem] + offset: int + limit: int + has_more: bool + + @classmethod + def from_page(cls, page: PolicyPage) -> "PolicyPageResult": + return cls( + items=[PolicyItem.from_read(item) for item in page.items], + offset=page.offset, + limit=page.limit, + has_more=page.has_more, + ) + + +ResultT = TypeVar("ResultT") + + +class PolicySuccessResponse(StrictPolicyAPIModel, Generic[ResultT]): + status: Literal["ok"] = "ok" + message: None = None + result: ResultT + + +class PolicyDetailResponse(PolicySuccessResponse[PolicyDetailResult]): + pass + + +class PolicyPageResponse(PolicySuccessResponse[PolicyPageResult]): + pass + + +class PolicyErrorResponse(StrictPolicyAPIModel): + status: Literal["error"] = "error" + message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] + + +POLICY_ERROR_RESPONSES = { + 400: { + "model": PolicyErrorResponse, + "description": "The policy content or country selection is invalid.", + }, + 404: { + "model": PolicyErrorResponse, + "description": "The selected policy or catalog does not exist.", + }, + 409: { + "model": PolicyErrorResponse, + "description": "Immutable policy identity conflicts with stored state.", + }, + 413: { + "model": PolicyErrorResponse, + "description": "The policy request body exceeds 1 MiB.", + }, + 422: { + "model": PolicyErrorResponse, + "description": "The request does not match the policy schema.", + }, + 500: { + "model": PolicyErrorResponse, + "description": "Stored policy integrity validation failed.", + }, + 503: { + "model": PolicyErrorResponse, + "description": "Supabase policy persistence is unavailable.", + }, +} diff --git a/policyengine_api/data/v2/policies/canonicalization.py b/policyengine_api/data/v2/policies/canonicalization.py new file mode 100644 index 000000000..9b1b0df9e --- /dev/null +++ b/policyengine_api/data/v2/policies/canonicalization.py @@ -0,0 +1,107 @@ +"""Versioned canonical content identity for immutable v2 policies.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from decimal import Decimal +import hashlib +import json +from typing import Any + +from policyengine_api.data.v2.policies.schemas import ResolvedPolicyCreateCommand + + +POLICY_CANONICALIZATION_VERSION = 1 + + +@dataclass(frozen=True) +class CanonicalPolicyContent: + """Canonical bytes and SHA-256 identity for one resolved policy.""" + + version: int + document: bytes + content_hash: str + + +def _canonical_number(value: int | float) -> str: + number = Decimal(str(value)) + if number.is_zero(): + return "0" + rendered = format(number, "f") + if "." in rendered: + rendered = rendered.rstrip("0").rstrip(".") + return rendered + + +def _canonical_json(value: Any) -> str: + if value is None: + return "null" + if type(value) is bool: + return "true" if value else "false" + if type(value) in {int, float}: + return _canonical_number(value) + if type(value) is str: + return json.dumps(value, ensure_ascii=True, allow_nan=False) + if type(value) is list: + return "[" + ",".join(_canonical_json(item) for item in value) + "]" + if type(value) is dict: + members = ( + f"{json.dumps(key, ensure_ascii=True)}:{_canonical_json(value[key])}" + for key in sorted(value) + ) + return "{" + ",".join(members) + "}" + raise TypeError("canonical policy content contains a non-JSON value") + + +def canonical_utc_datetime(value: datetime) -> str: + """Render an aware datetime as fixed-width UTC with a trailing Z.""" + + utc_value = value.astimezone(timezone.utc) + return utc_value.isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def canonical_policy_document(command: ResolvedPolicyCreateCommand) -> bytes: + """Serialize only immutable policy content in deterministic order.""" + + parameter_values = sorted( + command.parameter_values, + key=lambda value: ( + str(value.parameter_id), + canonical_utc_datetime(value.start_date), + "" if value.end_date is None else canonical_utc_datetime(value.end_date), + ), + ) + document = { + "canonicalization_version": POLICY_CANONICALIZATION_VERSION, + "country_id": command.country_id, + "tax_benefit_model_id": str(command.tax_benefit_model_id), + "tax_benefit_model_version_id": str(command.tax_benefit_model_version_id), + "parameter_values": [ + { + "parameter_id": str(value.parameter_id), + "value": value.value, + "start_date": canonical_utc_datetime(value.start_date), + "end_date": ( + None + if value.end_date is None + else canonical_utc_datetime(value.end_date) + ), + } + for value in parameter_values + ], + } + return _canonical_json(document).encode("ascii") + + +def canonicalize_policy( + command: ResolvedPolicyCreateCommand, +) -> CanonicalPolicyContent: + """Return versioned canonical bytes and their lowercase SHA-256 digest.""" + + document = canonical_policy_document(command) + return CanonicalPolicyContent( + version=POLICY_CANONICALIZATION_VERSION, + document=document, + content_hash=hashlib.sha256(document).hexdigest(), + ) diff --git a/policyengine_api/data/v2/policies/catalog.py b/policyengine_api/data/v2/policies/catalog.py new file mode 100644 index 000000000..d9e23741f --- /dev/null +++ b/policyengine_api/data/v2/policies/catalog.py @@ -0,0 +1,77 @@ +"""Catalog binding for immutable v2 policy commands.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, select + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v2.catalog.catalog_selection import select_catalog +from policyengine_api.data.v2.models import Parameter +from policyengine_api.data.v2.policies.schemas import ( + PolicyCreateCommand, + ResolvedPolicyCreateCommand, +) + + +class PolicyCatalogValidationError(ValueError): + """Raised when policy content does not belong to the selected catalog.""" + + +def _version_parameter_ids( + session: Session, + *, + model_version_id: UUID, + requested_ids: set[UUID], +) -> set[UUID]: + if not requested_ids: + return set() + return set( + session.exec( + select(Parameter.id).where( + Parameter.tax_benefit_model_version_id == model_version_id, + Parameter.id.in_(requested_ids), + ) + ).all() + ) + + +def resolve_policy_catalog( + session: Session, + command: PolicyCreateCommand, + *, + policyengine_version: str | None = None, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> ResolvedPolicyCreateCommand: + """Bind validated content to one exact initialized catalog.""" + + selected = select_catalog( + session, + country_id=command.country_id, + running_policyengine_version=running_policyengine_version, + policyengine_version=policyengine_version, + ) + if command.tax_benefit_model_id != selected.model.id: + raise PolicyCatalogValidationError( + "tax_benefit_model_id does not match the selected country catalog" + ) + + requested_parameter_ids = {value.parameter_id for value in command.parameter_values} + resolved_parameter_ids = _version_parameter_ids( + session, + model_version_id=selected.model_version.id, + requested_ids=requested_parameter_ids, + ) + if resolved_parameter_ids != requested_parameter_ids: + raise PolicyCatalogValidationError( + "every parameter_id must belong to the selected model version" + ) + + return ResolvedPolicyCreateCommand( + country_id=command.country_id, + tax_benefit_model_id=selected.model.id, + tax_benefit_model_version_id=selected.model_version.id, + policyengine_version=selected.policyengine_version, + parameter_values=command.parameter_values, + ) diff --git a/policyengine_api/data/v2/policies/legacy.py b/policyengine_api/data/v2/policies/legacy.py new file mode 100644 index 000000000..053aac23f --- /dev/null +++ b/policyengine_api/data/v2/policies/legacy.py @@ -0,0 +1,286 @@ +"""Translation of committed v1 policy snapshots into v2 policy commands.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date, datetime, time, timezone +from typing import Annotated +from uuid import UUID + +from policyengine_core.periods import period as parse_policyengine_period +from pydantic import Field, field_validator +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, select + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION +from policyengine_api.data.v2.catalog.catalog_selection import select_catalog +from policyengine_api.data.v2.models import LegacyPolicyMapping, Parameter +from policyengine_api.data.v2.policies.catalog import resolve_policy_catalog +from policyengine_api.data.v2.policies.persistence import persist_resolved_policy +from policyengine_api.data.v2.policies.schemas import ( + PolicyCreateCommand, + ResolvedPolicyCreateCommand, + StrictJsonValue, + StrictPolicyCommand, +) +from policyengine_api.query_parameters import CountryId + + +class LegacyPolicyTranslationError(ValueError): + """Raised when committed v1 content cannot be interpreted exactly.""" + + +class LegacyPolicyMappingIntegrityError(RuntimeError): + """Raised when one immutable v1 identity changes or maps inconsistently.""" + + +@dataclass(frozen=True) +class LegacyPolicyPersistenceResult: + """Destination identity and insertion outcomes for one mirror attempt.""" + + policy_id: UUID + policy_created: bool + mapping_created: bool + + +class LegacyPolicySnapshot(StrictPolicyCommand): + """Detached committed fields required by the v2 policy mirror.""" + + country_id: CountryId + legacy_policy_id: Annotated[int, Field(ge=0)] + label: Annotated[str, Field(max_length=255)] | None = None + api_version: Annotated[str, Field(min_length=1, max_length=255)] + policy_json: StrictJsonValue + source_policy_hash: Annotated[str, Field(min_length=1, max_length=255)] + + @field_validator("policy_json") + @classmethod + def require_parameter_mapping(cls, value: object) -> object: + if type(value) is not dict: + raise ValueError("legacy policy_json must be a parameter-path object") + return value + + +def _utc_midnight(value: str) -> datetime: + try: + parsed = date.fromisoformat(value) + except ValueError as error: + raise LegacyPolicyTranslationError( + f"legacy period date {value!r} is invalid" + ) from error + return datetime.combine(parsed, time.min, tzinfo=timezone.utc) + + +def parse_legacy_period(value: str) -> tuple[datetime, datetime]: + """Translate one legacy period key to inclusive UTC endpoints.""" + + if not value or value != value.strip(): + raise LegacyPolicyTranslationError("legacy period must be non-empty") + if "." in value: + parts = value.split(".") + if len(parts) != 2: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} must contain one date range" + ) + start_date, end_date = (_utc_midnight(item) for item in parts) + else: + try: + parsed = parse_policyengine_period(value) + start_date = _utc_midnight(str(parsed.start)) + end_date = _utc_midnight(str(parsed.stop)) + except (TypeError, ValueError) as error: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} is invalid" + ) from error + if end_date < start_date: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} ends before it starts" + ) + return start_date, end_date + + +def _parameters_by_name( + session: Session, + *, + model_version_id, + names: set[str], +) -> dict[str, Parameter]: + if not names: + return {} + parameters = session.exec( + select(Parameter).where( + Parameter.tax_benefit_model_version_id == model_version_id, + Parameter.name.in_(names), + ) + ).all() + return {parameter.name: parameter for parameter in parameters} + + +def translate_legacy_policy( + session: Session, + snapshot: LegacyPolicySnapshot, + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + country_package_versions: Mapping[str, str] = COUNTRY_PACKAGE_VERSIONS, +) -> ResolvedPolicyCreateCommand: + """Resolve a committed legacy reform through the exact running catalog.""" + + expected_api_version = country_package_versions.get(snapshot.country_id) + if expected_api_version is None or snapshot.api_version != expected_api_version: + raise LegacyPolicyTranslationError( + "legacy policy api_version does not match the running country package" + ) + selected = select_catalog( + session, + country_id=snapshot.country_id, + running_policyengine_version=running_policyengine_version, + ) + policy_json = snapshot.policy_json + assert isinstance(policy_json, dict) + parameter_names = set(policy_json) + parameters = _parameters_by_name( + session, + model_version_id=selected.model_version.id, + names=parameter_names, + ) + if set(parameters) != parameter_names: + raise LegacyPolicyTranslationError( + "every legacy parameter path must exist in the running catalog" + ) + + parameter_values: list[dict[str, object]] = [] + for parameter_name in sorted(parameter_names): + period_values = policy_json[parameter_name] + if type(period_values) is not dict: + raise LegacyPolicyTranslationError( + f"legacy parameter {parameter_name!r} must contain period/value entries" + ) + for period_name, value in sorted(period_values.items()): + if type(period_name) is not str: + raise LegacyPolicyTranslationError( + f"legacy parameter {parameter_name!r} has a non-string period" + ) + start_date, end_date = parse_legacy_period(period_name) + parameter_values.append( + { + "parameter_id": parameters[parameter_name].id, + "value": value, + "start_date": start_date, + "end_date": end_date, + } + ) + + try: + command = PolicyCreateCommand( + country_id=snapshot.country_id, + tax_benefit_model_id=selected.model.id, + parameter_values=parameter_values, + ) + except ValueError as error: + raise LegacyPolicyTranslationError( + "legacy parameter periods or values conflict" + ) from error + return resolve_policy_catalog( + session, + command, + running_policyengine_version=running_policyengine_version, + ) + + +def _legacy_mapping( + session: Session, + snapshot: LegacyPolicySnapshot, + *, + lock: bool, +) -> LegacyPolicyMapping | None: + statement = select(LegacyPolicyMapping).where( + LegacyPolicyMapping.country_id == snapshot.country_id, + LegacyPolicyMapping.legacy_policy_id == snapshot.legacy_policy_id, + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def _verify_legacy_mapping( + mapping: LegacyPolicyMapping, + snapshot: LegacyPolicySnapshot, + *, + expected_policy_id: UUID | None = None, +) -> None: + if mapping.source_policy_hash != snapshot.source_policy_hash: + raise LegacyPolicyMappingIntegrityError( + "legacy policy identity was presented with a different source hash" + ) + if expected_policy_id is not None and mapping.policy_id != expected_policy_id: + raise LegacyPolicyMappingIntegrityError( + "legacy policy mapping does not match translated immutable content" + ) + + +def persist_legacy_policy( + session: Session, + snapshot: LegacyPolicySnapshot, + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + country_package_versions: Mapping[str, str] = COUNTRY_PACKAGE_VERSIONS, +) -> LegacyPolicyPersistenceResult: + """Translate, deduplicate, and map one v1 policy in the caller transaction.""" + + existing = _legacy_mapping(session, snapshot, lock=True) + if existing is not None: + _verify_legacy_mapping(existing, snapshot) + + command = translate_legacy_policy( + session, + snapshot, + running_policyengine_version=running_policyengine_version, + country_package_versions=country_package_versions, + ) + policy_result = persist_resolved_policy(session, command) + if existing is not None: + _verify_legacy_mapping( + existing, + snapshot, + expected_policy_id=policy_result.policy_id, + ) + return LegacyPolicyPersistenceResult( + policy_id=existing.policy_id, + policy_created=False, + mapping_created=False, + ) + + mapping_id = session.execute( + insert(LegacyPolicyMapping) + .values( + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + policy_id=policy_result.policy_id, + source_policy_hash=snapshot.source_policy_hash, + ) + .on_conflict_do_nothing(constraint="uq_legacy_policy_mappings_country_legacy") + .returning(LegacyPolicyMapping.id) + ).scalar_one_or_none() + if mapping_id is not None: + return LegacyPolicyPersistenceResult( + policy_id=policy_result.policy_id, + policy_created=policy_result.created, + mapping_created=True, + ) + + concurrent = _legacy_mapping(session, snapshot, lock=False) + if concurrent is None: + raise LegacyPolicyMappingIntegrityError( + "legacy policy mapping conflict did not resolve to a stored row" + ) + _verify_legacy_mapping( + concurrent, + snapshot, + expected_policy_id=policy_result.policy_id, + ) + return LegacyPolicyPersistenceResult( + policy_id=concurrent.policy_id, + policy_created=False, + mapping_created=False, + ) diff --git a/policyengine_api/data/v2/policies/persistence.py b/policyengine_api/data/v2/policies/persistence.py new file mode 100644 index 000000000..2e7786d8b --- /dev/null +++ b/policyengine_api/data/v2/policies/persistence.py @@ -0,0 +1,158 @@ +"""Conflict-aware PostgreSQL persistence for immutable v2 policies.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from uuid import UUID, uuid4 + +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, select + +from policyengine_api.data.v2.models import ( + ParameterValue, + Policy, + TaxBenefitModelVersion, +) +from policyengine_api.data.v2.policies.canonicalization import ( + CanonicalPolicyContent, + canonical_policy_document, + canonicalize_policy, +) +from policyengine_api.data.v2.policies.schemas import ResolvedPolicyCreateCommand + + +class PolicyPersistenceIntegrityError(RuntimeError): + """Raised when policy hash persistence no longer matches stored content.""" + + +class PolicyContentHashCollisionError(PolicyPersistenceIntegrityError): + """Raised when equal version/hash keys identify different canonical bytes.""" + + +@dataclass(frozen=True) +class PolicyPersistenceResult: + """Inserted or deduplicated immutable policy identity.""" + + policy_id: UUID + created: bool + + +def _insert_policy( + session: Session, + command: ResolvedPolicyCreateCommand, + content: CanonicalPolicyContent, +) -> UUID | None: + policy_id = uuid4() + statement = ( + insert(Policy) + .values( + id=policy_id, + country_id=command.country_id, + tax_benefit_model_id=command.tax_benefit_model_id, + tax_benefit_model_version_id=command.tax_benefit_model_version_id, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") + .returning(Policy.id) + ) + return session.execute(statement).scalar_one_or_none() + + +def _insert_parameter_values( + session: Session, + *, + policy_id: UUID, + command: ResolvedPolicyCreateCommand, +) -> None: + session.add_all( + [ + ParameterValue( + policy_id=policy_id, + dynamic_id=None, + parameter_id=value.parameter_id, + value_json=value.value, + start_date=value.start_date, + end_date=value.end_date, + ) + for value in command.parameter_values + ] + ) + session.flush() + + +def _stored_policy_command( + session: Session, + policy: Policy, +) -> ResolvedPolicyCreateCommand: + model_version = session.get( + TaxBenefitModelVersion, + policy.tax_benefit_model_version_id, + ) + if model_version is None: + raise PolicyPersistenceIntegrityError( + "stored policy references an absent model version" + ) + values = session.exec( + select(ParameterValue).where(ParameterValue.policy_id == policy.id) + ).all() + return ResolvedPolicyCreateCommand( + country_id=policy.country_id, + tax_benefit_model_id=policy.tax_benefit_model_id, + tax_benefit_model_version_id=policy.tax_benefit_model_version_id, + policyengine_version=model_version.version, + parameter_values=[ + { + "parameter_id": value.parameter_id, + "value": value.value_json, + "start_date": value.start_date, + "end_date": value.end_date, + } + for value in values + ], + ) + + +def _existing_policy_after_conflict( + session: Session, + content: CanonicalPolicyContent, +) -> Policy: + policy = session.exec( + select(Policy).where( + Policy.canonicalization_version == content.version, + Policy.content_hash == content.content_hash, + ) + ).one_or_none() + if policy is None: + raise PolicyPersistenceIntegrityError( + "policy hash conflict did not resolve to a stored policy" + ) + return policy + + +def persist_resolved_policy( + session: Session, + command: ResolvedPolicyCreateCommand, + *, + canonicalizer: Callable[ + [ResolvedPolicyCreateCommand], CanonicalPolicyContent + ] = canonicalize_policy, +) -> PolicyPersistenceResult: + """Insert one policy atomically or verify and return equivalent content.""" + + content = canonicalizer(command) + inserted_id = _insert_policy(session, command, content) + if inserted_id is not None: + _insert_parameter_values(session, policy_id=inserted_id, command=command) + return PolicyPersistenceResult(policy_id=inserted_id, created=True) + + existing = _existing_policy_after_conflict(session, content) + stored_document = canonical_policy_document( + _stored_policy_command(session, existing) + ) + if stored_document != content.document: + raise PolicyContentHashCollisionError( + "stored policy content differs for the same canonical version and hash" + ) + return PolicyPersistenceResult(policy_id=existing.id, created=False) diff --git a/policyengine_api/data/v2/policies/query.py b/policyengine_api/data/v2/policies/query.py new file mode 100644 index 000000000..5852a441e --- /dev/null +++ b/policyengine_api/data/v2/policies/query.py @@ -0,0 +1,145 @@ +"""Country-scoped immutable v2 policy read operations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import Any +from uuid import UUID + +from sqlmodel import Session, select + +from policyengine_api.data.v2.models import Parameter, ParameterValue, Policy + + +class PolicyNotFoundError(LookupError): + """Raised when a policy UUID is absent from the selected country.""" + + +@dataclass(frozen=True) +class PolicyParameterValueRead: + id: UUID + parameter_id: UUID + parameter_name: str + value: Any + start_date: datetime + end_date: datetime | None + + +@dataclass(frozen=True) +class PolicyRead: + id: UUID + country_id: str + tax_benefit_model_id: UUID + tax_benefit_model_version_id: UUID + created_at: datetime + updated_at: datetime + parameter_values: tuple[PolicyParameterValueRead, ...] + + +@dataclass(frozen=True) +class PolicyPage: + items: tuple[PolicyRead, ...] + offset: int + limit: int + has_more: bool + + +def _parameter_values_by_policy( + session: Session, + policy_ids: list[UUID], +) -> dict[UUID, tuple[PolicyParameterValueRead, ...]]: + grouped: dict[UUID, list[PolicyParameterValueRead]] = { + policy_id: [] for policy_id in policy_ids + } + if not policy_ids: + return {} + rows = session.exec( + select(ParameterValue, Parameter.name) + .join(Parameter, Parameter.id == ParameterValue.parameter_id) + .where(ParameterValue.policy_id.in_(policy_ids)) + .order_by( + Parameter.name, + ParameterValue.start_date, + ParameterValue.id, + ) + ).all() + for value, parameter_name in rows: + if value.policy_id is None: + continue + grouped[value.policy_id].append( + PolicyParameterValueRead( + id=value.id, + parameter_id=value.parameter_id, + parameter_name=parameter_name, + value=value.value_json, + start_date=value.start_date, + end_date=value.end_date, + ) + ) + return {policy_id: tuple(values) for policy_id, values in grouped.items()} + + +def _policy_read( + policy: Policy, + values: dict[UUID, tuple[PolicyParameterValueRead, ...]], +) -> PolicyRead: + return PolicyRead( + id=policy.id, + country_id=policy.country_id, + tax_benefit_model_id=policy.tax_benefit_model_id, + tax_benefit_model_version_id=policy.tax_benefit_model_version_id, + created_at=policy.created_at, + updated_at=policy.updated_at, + parameter_values=values.get(policy.id, ()), + ) + + +def read_policy( + session: Session, + *, + country_id: str, + policy_id: UUID, +) -> PolicyRead: + """Read one complete policy only under its stored country.""" + + policy = session.exec( + select(Policy).where( + Policy.id == policy_id, + Policy.country_id == country_id, + ) + ).one_or_none() + if policy is None: + raise PolicyNotFoundError(f"policy {policy_id} was not found") + values = _parameter_values_by_policy(session, [policy.id]) + return _policy_read(policy, values) + + +def list_policies( + session: Session, + *, + country_id: str, + tax_benefit_model_id: UUID | None = None, + offset: int = 0, + limit: int = 100, +) -> PolicyPage: + """Read one deterministic bounded page with optional exact model filtering.""" + + statement = select(Policy).where(Policy.country_id == country_id) + if tax_benefit_model_id is not None: + statement = statement.where(Policy.tax_benefit_model_id == tax_benefit_model_id) + rows = session.exec( + statement.order_by(Policy.created_at, Policy.id).offset(offset).limit(limit + 1) + ).all() + has_more = len(rows) > limit + policies = rows[:limit] + values = _parameter_values_by_policy( + session, + [policy.id for policy in policies], + ) + return PolicyPage( + items=tuple(_policy_read(policy, values) for policy in policies), + offset=offset, + limit=limit, + has_more=has_more, + ) diff --git a/policyengine_api/data/v2/policies/schemas.py b/policyengine_api/data/v2/policies/schemas.py new file mode 100644 index 000000000..ddd3d127f --- /dev/null +++ b/policyengine_api/data/v2/policies/schemas.py @@ -0,0 +1,138 @@ +"""Route-independent commands for immutable v2 policy creation.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import math +from typing import Annotated, Any +from uuid import UUID + +from pydantic import ( + AfterValidator, + BaseModel, + BeforeValidator, + ConfigDict, + Field, + JsonValue, + field_validator, + model_validator, +) + +from policyengine_api.query_parameters import CountryId, PolicyEngineVersion + + +MAXIMUM_POLICY_PARAMETER_VALUES = 1_000 +MAXIMUM_JSON_NESTING = 100 + + +def _require_json_value( + value: Any, + *, + depth: int = 0, + containers: frozenset[int] = frozenset(), +) -> Any: + if depth > MAXIMUM_JSON_NESTING: + raise ValueError("JSON values must not exceed 100 nested containers") + if value is None or type(value) in {str, bool, int}: + return value + if type(value) is float: + if not math.isfinite(value): + raise ValueError("JSON numbers must be finite") + return value + if type(value) not in {list, dict}: + raise ValueError("value must contain only standards-compliant JSON types") + identity = id(value) + if identity in containers: + raise ValueError("JSON values must not contain reference cycles") + nested_containers = containers | {identity} + if type(value) is list: + for item in value: + _require_json_value( + item, + depth=depth + 1, + containers=nested_containers, + ) + return value + for key, item in value.items(): + if type(key) is not str: + raise ValueError("JSON object keys must be strings") + _require_json_value( + item, + depth=depth + 1, + containers=nested_containers, + ) + return value + + +def _normalize_utc(value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("effective dates must include a UTC offset") + return value.astimezone(timezone.utc) + + +StrictJsonValue = Annotated[ + JsonValue, + BeforeValidator(_require_json_value), +] +UtcDateTime = Annotated[datetime, AfterValidator(_normalize_utc)] + + +class StrictPolicyCommand(BaseModel): + """Reject undeclared policy input and non-finite numeric coercion.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, frozen=True) + + +class PolicyParameterValueCommand(StrictPolicyCommand): + """One normalized effective value for a catalog parameter UUID.""" + + parameter_id: UUID + value: StrictJsonValue + start_date: UtcDateTime + end_date: UtcDateTime | None = None + + @model_validator(mode="after") + def validate_period(self) -> "PolicyParameterValueCommand": + if self.end_date is not None and self.end_date < self.start_date: + raise ValueError("end_date must not precede start_date") + return self + + +class PolicyCreateCommand(StrictPolicyCommand): + """Complete immutable content accepted from native and translated inputs.""" + + country_id: CountryId + tax_benefit_model_id: UUID + parameter_values: Annotated[ + list[PolicyParameterValueCommand], + Field(max_length=MAXIMUM_POLICY_PARAMETER_VALUES), + ] + + @field_validator("parameter_values") + @classmethod + def reject_duplicate_effective_values( + cls, + values: list[PolicyParameterValueCommand], + ) -> list[PolicyParameterValueCommand]: + identities: set[tuple[UUID, datetime]] = set() + for value in values: + identity = (value.parameter_id, value.start_date) + if identity in identities: + raise ValueError( + "parameter_values must not repeat a parameter_id/start_date" + ) + identities.add(identity) + return values + + +class NativePolicyCreateCommand(PolicyCreateCommand): + """Native content plus its optional catalog-version selection.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class ResolvedPolicyCreateCommand(PolicyCreateCommand): + """Validated content bound to one exact initialized catalog version.""" + + policyengine_version: PolicyEngineVersion + tax_benefit_model_version_id: UUID diff --git a/policyengine_api/data/v2/policies/service.py b/policyengine_api/data/v2/policies/service.py new file mode 100644 index 000000000..9eabf2e6f --- /dev/null +++ b/policyengine_api/data/v2/policies/service.py @@ -0,0 +1,109 @@ +"""Session-owning application service for native and mirrored v2 policies.""" + +from __future__ import annotations + +from dataclasses import dataclass +from uuid import UUID + +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v2.policies.catalog import resolve_policy_catalog +from policyengine_api.data.v2.policies.legacy import ( + LegacyPolicyPersistenceResult, + LegacyPolicySnapshot, + persist_legacy_policy, +) +from policyengine_api.data.v2.policies.persistence import persist_resolved_policy +from policyengine_api.data.v2.policies.query import ( + PolicyPage, + PolicyRead, + list_policies, + read_policy, +) +from policyengine_api.data.v2.policies.schemas import ( + NativePolicyCreateCommand, + PolicyCreateCommand, +) + + +@dataclass(frozen=True) +class NativePolicyCreation: + """Complete policy read plus whether this request inserted it.""" + + item: PolicyRead + created: bool + + +class V2PolicyService: + """Own transaction boundaries for immutable policy operations.""" + + def __init__( + self, + session_factory: sessionmaker[Session], + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + ) -> None: + self._sessions = session_factory + self._running_policyengine_version = running_policyengine_version + + def create_policy( + self, + command: NativePolicyCreateCommand, + ) -> NativePolicyCreation: + content = PolicyCreateCommand.model_validate( + command.model_dump(exclude={"policyengine_version"}) + ) + with self._sessions.begin() as session: + resolved = resolve_policy_catalog( + session, + content, + policyengine_version=command.policyengine_version, + running_policyengine_version=self._running_policyengine_version, + ) + persisted = persist_resolved_policy(session, resolved) + item = read_policy( + session, + country_id=command.country_id, + policy_id=persisted.policy_id, + ) + return NativePolicyCreation(item=item, created=persisted.created) + + def get_policy(self, *, country_id: str, policy_id: UUID) -> PolicyRead: + with self._sessions() as session: + return read_policy( + session, + country_id=country_id, + policy_id=policy_id, + ) + + def list_policies( + self, + *, + country_id: str, + tax_benefit_model_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> PolicyPage: + with self._sessions() as session: + return list_policies( + session, + country_id=country_id, + tax_benefit_model_id=tax_benefit_model_id, + offset=offset, + limit=limit, + ) + + def mirror_legacy_policy( + self, + snapshot: LegacyPolicySnapshot, + ) -> LegacyPolicyPersistenceResult: + """Mirror one committed v1 row in one Supabase transaction.""" + + with self._sessions.begin() as session: + return persist_legacy_policy( + session, + snapshot, + running_policyengine_version=self._running_policyengine_version, + ) diff --git a/policyengine_api/data/v2/policy_migration_qualification.py b/policyengine_api/data/v2/policy_migration_qualification.py new file mode 100644 index 000000000..b5e295a92 --- /dev/null +++ b/policyengine_api/data/v2/policy_migration_qualification.py @@ -0,0 +1,177 @@ +"""Read-only qualification for dormant v2 policy data.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +import json +import sys +from typing import Protocol + +from sqlalchemy import Connection, Engine, func, select +from sqlalchemy.pool import NullPool +from sqlmodel import create_engine + +from policyengine_api.data.v2.models import ParameterValue, Policy, UserPolicy +from policyengine_api.data.v2.settings import ( + V2ConfigurationError, + V2DatabaseSettings, + load_v2_migration_database_settings, +) + + +class ScalarExecutor(Protocol): + """Minimal database interface required by the row-count queries.""" + + def scalar(self, statement: object) -> object | None: + """Return the first column from the first result row.""" + + +@dataclass(frozen=True) +class PolicyDataCounts: + """Counts of predecessor policy data that the migration would replace.""" + + policies: int + policy_parameter_values: int + user_policies: int + + @property + def total(self) -> int: + return self.policies + self.policy_parameter_values + self.user_policies + + def as_dict(self) -> dict[str, int]: + return { + "policies": self.policies, + "policy_parameter_values": self.policy_parameter_values, + "user_policies": self.user_policies, + } + + +@dataclass(frozen=True) +class PolicyMigrationQualification: + """Non-secret evidence that one configured Supabase target is empty.""" + + environment: str + project_ref: str + counts: PolicyDataCounts + + def as_dict(self) -> dict[str, object]: + return { + "outcome": "ok", + "environment": self.environment, + "project_ref": self.project_ref, + "counts": self.counts.as_dict(), + } + + +class RetainedPolicyDataError(RuntimeError): + """Raised when a target contains policy data requiring preservation.""" + + def __init__(self, counts: PolicyDataCounts) -> None: + self.counts = counts + super().__init__( + "the configured Supabase target contains retained v2 policy data " + f"(policies={counts.policies}, " + f"policy_parameter_values={counts.policy_parameter_values}, " + f"user_policies={counts.user_policies}); migration stopped without " + "modifying data. Use an empty target or obtain a reviewed data-" + "preservation plan before retrying" + ) + + +def read_policy_data_counts(executor: ScalarExecutor) -> PolicyDataCounts: + """Count only policy-owned rows; canonical catalog values are excluded.""" + + return PolicyDataCounts( + policies=int(executor.scalar(select(func.count()).select_from(Policy)) or 0), + policy_parameter_values=int( + executor.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id.is_not(None)) + ) + or 0 + ), + user_policies=int( + executor.scalar(select(func.count()).select_from(UserPolicy)) or 0 + ), + ) + + +def require_no_retained_policy_data(counts: PolicyDataCounts) -> None: + """Stop the migration when any predecessor policy row requires a decision.""" + + if counts.total: + raise RetainedPolicyDataError(counts) + + +def build_qualification_engine(settings: V2DatabaseSettings) -> Engine: + """Build an isolated connection pool for one qualification attempt.""" + + return create_engine(settings.connection.url, poolclass=NullPool) + + +def _qualify_connection(connection: Connection) -> PolicyDataCounts: + transaction = connection.begin() + try: + connection.exec_driver_sql("SET TRANSACTION READ ONLY") + counts = read_policy_data_counts(connection) + require_no_retained_policy_data(counts) + return counts + finally: + transaction.rollback() + + +def qualify_policy_migration_target( + environ: Mapping[str, str] | None = None, + *, + engine_builder: Callable[[V2DatabaseSettings], Engine] = ( + build_qualification_engine + ), +) -> PolicyMigrationQualification: + """Qualify the configured Supabase target without changing it.""" + + settings = load_v2_migration_database_settings(environ) + engine = engine_builder(settings) + try: + with engine.connect() as connection: + counts = _qualify_connection(connection) + finally: + engine.dispose() + return PolicyMigrationQualification( + environment=settings.target.environment, + project_ref=settings.target.project_ref, + counts=counts, + ) + + +def _error_payload(error: Exception) -> dict[str, object]: + safe_errors = (V2ConfigurationError, RetainedPolicyDataError) + message = ( + str(error) + if isinstance(error, safe_errors) + else "v2 policy migration qualification failed unexpectedly" + ) + return { + "outcome": "error", + "error": { + "type": type(error).__name__, + "message": message, + }, + } + + +def main() -> int: + """Run qualification and return a shell-compatible status.""" + + try: + evidence = qualify_policy_migration_target() + except Exception as error: # noqa: BLE001 - command must emit safe evidence + print(json.dumps(_error_payload(error), sort_keys=True), file=sys.stderr) + return 1 + print(json.dumps(evidence.as_dict(), sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/policyengine_api/data/v2/settings.py b/policyengine_api/data/v2/settings.py index a75b83def..06920ac9c 100644 --- a/policyengine_api/data/v2/settings.py +++ b/policyengine_api/data/v2/settings.py @@ -32,6 +32,7 @@ ENVIRONMENT_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,31}$") SECRET_RESOURCE_PATTERN = re.compile(r"^projects/[^/]+/secrets/[^/]+/versions/[^/]+$") SUPABASE_DATABASE_NAME = "postgres" +SUPABASE_TRANSACTION_POOLER_PORT = 6543 class V2ConfigurationError(RuntimeError): @@ -249,11 +250,18 @@ def load_v2_runtime_database_settings( values, secret_loader=secret_loader or _load_secret_from_secret_manager, ) - return _database_settings( + settings = _database_settings( raw_url, setting_name=V2_RUNTIME_DATABASE_URL, environ=values, ) + if settings.connection.url.port == SUPABASE_TRANSACTION_POOLER_PORT: + raise V2ConfigurationError( + f"{V2_RUNTIME_DATABASE_URL} must use a direct connection or " + "Supavisor session mode on port 5432 so the runtime statement " + "timeout is applied" + ) + return settings def load_v2_migration_database_settings( diff --git a/policyengine_api/data/v2/user_policies/__init__.py b/policyengine_api/data/v2/user_policies/__init__.py new file mode 100644 index 000000000..2e52406fe --- /dev/null +++ b/policyengine_api/data/v2/user_policies/__init__.py @@ -0,0 +1 @@ +"""Native v2 user-policy association operations.""" diff --git a/policyengine_api/data/v2/user_policies/api_schemas.py b/policyengine_api/data/v2/user_policies/api_schemas.py new file mode 100644 index 000000000..a1b23ccbd --- /dev/null +++ b/policyengine_api/data/v2/user_policies/api_schemas.py @@ -0,0 +1,118 @@ +"""Strict HTTP schemas for native v2 user-policy associations.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Annotated, Generic, Literal, TypeVar +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, StringConstraints + +from policyengine_api.data.v2.user_policies.query import ( + UserPolicyPage, + UserPolicyRead, +) +from policyengine_api.data.v2.user_policies.schemas import ( + UserPolicyCreateCommand, + UserPolicyPatchCommand, +) +from policyengine_api.query_parameters import CountryId, ResourceId, UserId + + +class StrictUserPolicyAPIModel(BaseModel): + """Strict association contract with dataclass conversion support.""" + + model_config = ConfigDict(extra="forbid", from_attributes=True) + + +class UserPolicyCreateRequest(UserPolicyCreateCommand): + """Association identity, immutable link fields, and presentation fields.""" + + +class UserPolicyPatchRequest(UserPolicyPatchCommand): + """Explicitly supplied mutable presentation fields.""" + + +class UserPolicyItem(StrictUserPolicyAPIModel): + id: UUID + country_id: CountryId + user_id: UserId + policy_id: ResourceId + name: str | None + description: str | None + created_at: datetime + updated_at: datetime + + @classmethod + def from_read(cls, item: UserPolicyRead) -> "UserPolicyItem": + return cls.model_validate(item) + + +class UserPolicyDetailResult(StrictUserPolicyAPIModel): + item: UserPolicyItem + + +class UserPolicyPageResult(StrictUserPolicyAPIModel): + items: list[UserPolicyItem] + offset: int + limit: int + has_more: bool + + @classmethod + def from_page(cls, page: UserPolicyPage) -> "UserPolicyPageResult": + return cls( + items=[UserPolicyItem.from_read(item) for item in page.items], + offset=page.offset, + limit=page.limit, + has_more=page.has_more, + ) + + +ResultT = TypeVar("ResultT") + + +class UserPolicySuccessResponse(StrictUserPolicyAPIModel, Generic[ResultT]): + status: Literal["ok"] = "ok" + message: None = None + result: ResultT + + +class UserPolicyDetailResponse(UserPolicySuccessResponse[UserPolicyDetailResult]): + pass + + +class UserPolicyPageResponse(UserPolicySuccessResponse[UserPolicyPageResult]): + pass + + +class UserPolicyErrorResponse(StrictUserPolicyAPIModel): + status: Literal["error"] = "error" + message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] + + +USER_POLICY_ERROR_RESPONSES = { + 400: { + "model": UserPolicyErrorResponse, + "description": "Association content or country selection is invalid.", + }, + 404: { + "model": UserPolicyErrorResponse, + "description": "The selected policy or association does not exist.", + }, + 409: { + "model": UserPolicyErrorResponse, + "description": "Association state conflicts with stored state.", + }, + 422: { + "model": UserPolicyErrorResponse, + "description": "The request does not match the association schema.", + }, + 500: { + "model": UserPolicyErrorResponse, + "description": "The association operation could not be completed.", + }, + 503: { + "model": UserPolicyErrorResponse, + "description": "Supabase association persistence is unavailable.", + }, +} diff --git a/policyengine_api/data/v2/user_policies/legacy.py b/policyengine_api/data/v2/user_policies/legacy.py new file mode 100644 index 000000000..4321969e5 --- /dev/null +++ b/policyengine_api/data/v2/user_policies/legacy.py @@ -0,0 +1,278 @@ +"""Projection and durable mapping of committed v1 saved-policy rows.""" + +from __future__ import annotations + +from dataclasses import dataclass +from hashlib import sha256 +import json +from typing import Annotated +from uuid import UUID + +from pydantic import Field +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, select + +from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy +from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.data.v2.policies.legacy import ( + LegacyPolicySnapshot, + persist_legacy_policy, +) +from policyengine_api.data.v2.policies.schemas import StrictPolicyCommand +from policyengine_api.data.v2.user_policies.schemas import UserPolicyCreateCommand +from policyengine_api.query_parameters import CountryId, UserId + + +USER_POLICY_FINGERPRINT_VERSION = 1 + + +class LegacyUserPolicyIntegrityError(RuntimeError): + """Raised when source, policy, association, or mapping identity conflicts.""" + + +class LegacyUserPolicySnapshot(StrictPolicyCommand): + """Detached complete committed v1 saved-policy row.""" + + country_id: CountryId + legacy_user_policy_id: Annotated[int, Field(ge=0)] + reform_id: Annotated[int, Field(ge=0)] + reform_label: Annotated[str, Field(max_length=255)] | None = None + baseline_id: Annotated[int, Field(ge=0)] + baseline_label: Annotated[str, Field(max_length=255)] | None = None + user_id: UserId + year: Annotated[str, Field(max_length=32)] + geography: Annotated[str, Field(max_length=255)] + dataset: Annotated[str, Field(max_length=255)] | None = None + number_of_provisions: Annotated[int, Field(ge=0)] + api_version: Annotated[str, Field(max_length=32)] + added_date: int + updated_date: int + budgetary_impact: Annotated[str, Field(max_length=255)] | None = None + type: Annotated[str, Field(max_length=255)] | None = None + + +@dataclass(frozen=True) +class LegacyUserPolicyPersistenceResult: + association_id: UUID + policy_id: UUID + association_created: bool + association_updated: bool + mapping_created: bool + + +def fingerprint_legacy_user_policy(snapshot: LegacyUserPolicySnapshot) -> str: + """Hash every committed source field through deterministic JSON.""" + + document = { + "fingerprint_version": USER_POLICY_FINGERPRINT_VERSION, + **snapshot.model_dump(mode="json"), + } + encoded = json.dumps( + document, + ensure_ascii=True, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def project_legacy_user_policy( + snapshot: LegacyUserPolicySnapshot, + *, + policy_id: UUID, +) -> UserPolicyCreateCommand: + """Map v1 presentation data onto an association, never core policy content.""" + + return UserPolicyCreateCommand( + country_id=snapshot.country_id, + user_id=snapshot.user_id, + policy_id=policy_id, + name=snapshot.reform_label, + description=None, + ) + + +def _mapping( + session: Session, + snapshot: LegacyUserPolicySnapshot, + *, + lock: bool, +) -> LegacyUserPolicyMapping | None: + statement = select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.country_id == snapshot.country_id, + LegacyUserPolicyMapping.legacy_user_policy_id == snapshot.legacy_user_policy_id, + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def _mapped_association( + session: Session, + mapping: LegacyUserPolicyMapping, +) -> UserPolicy: + association = session.exec( + select(UserPolicy).where( + UserPolicy.id == mapping.user_policy_id, + UserPolicy.country_id == mapping.country_id, + ) + ).one_or_none() + if association is None: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping has no association" + ) + return association + + +def _apply_existing_mapping( + session: Session, + *, + mapping: LegacyUserPolicyMapping, + snapshot: LegacyUserPolicySnapshot, + fingerprint: str, + policy_id: UUID, + changed_fields: frozenset[str], + source_revision: int, +) -> LegacyUserPolicyPersistenceResult: + association = _mapped_association(session, mapping) + if ( + association.policy_id != policy_id + or association.country_id != snapshot.country_id + or association.user_id != snapshot.user_id + ): + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping conflicts with immutable association fields" + ) + if mapping.fingerprint_version != USER_POLICY_FINGERPRINT_VERSION: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy fingerprint version is unsupported" + ) + if source_revision < mapping.last_applied_source_revision: + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=False, + mapping_created=False, + ) + if source_revision == mapping.last_applied_source_revision: + if mapping.fingerprint_sha256 != fingerprint: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy revision conflicts with its stored fingerprint" + ) + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=False, + mapping_created=False, + ) + if source_revision != mapping.last_applied_source_revision + 1: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy revision has an unapplied predecessor" + ) + + association_updated = ( + "reform_label" in changed_fields and association.name != snapshot.reform_label + ) + if association_updated: + association.name = snapshot.reform_label + association.updated_at = utc_now() + session.add(association) + mapping.fingerprint_sha256 = fingerprint + mapping.last_applied_source_revision = source_revision + mapping.updated_at = utc_now() + session.add(mapping) + session.flush() + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=association_updated, + mapping_created=False, + ) + + +def persist_legacy_user_policy( + session: Session, + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str] = frozenset(), +) -> LegacyUserPolicyPersistenceResult: + """Ensure reform, association, and both mappings in the caller transaction.""" + + if source_revision <= 0: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy source revision must be positive" + ) + if ( + snapshot.country_id != reform_snapshot.country_id + or snapshot.reform_id != reform_snapshot.legacy_policy_id + ): + raise LegacyUserPolicyIntegrityError( + "saved policy does not reference the supplied reform snapshot" + ) + policy_result = persist_legacy_policy(session, reform_snapshot) + fingerprint = fingerprint_legacy_user_policy(snapshot) + existing = _mapping(session, snapshot, lock=True) + if existing is not None: + return _apply_existing_mapping( + session, + mapping=existing, + snapshot=snapshot, + fingerprint=fingerprint, + policy_id=policy_result.policy_id, + changed_fields=changed_fields, + source_revision=source_revision, + ) + + projection = project_legacy_user_policy( + snapshot, + policy_id=policy_result.policy_id, + ) + association = UserPolicy(**projection.model_dump()) + session.add(association) + session.flush() + mapping_id = session.execute( + insert(LegacyUserPolicyMapping) + .values( + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + user_policy_id=association.id, + last_applied_source_revision=source_revision, + fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, + fingerprint_sha256=fingerprint, + ) + .on_conflict_do_nothing( + constraint="uq_legacy_user_policy_mappings_country_legacy" + ) + .returning(LegacyUserPolicyMapping.id) + ).scalar_one_or_none() + if mapping_id is not None: + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_result.policy_id, + association_created=True, + association_updated=False, + mapping_created=True, + ) + + session.delete(association) + session.flush() + concurrent = _mapping(session, snapshot, lock=False) + if concurrent is None: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping conflict did not resolve to a stored row" + ) + return _apply_existing_mapping( + session, + mapping=concurrent, + snapshot=snapshot, + fingerprint=fingerprint, + policy_id=policy_result.policy_id, + changed_fields=changed_fields, + source_revision=source_revision, + ) diff --git a/policyengine_api/data/v2/user_policies/persistence.py b/policyengine_api/data/v2/user_policies/persistence.py new file mode 100644 index 000000000..55364f9bd --- /dev/null +++ b/policyengine_api/data/v2/user_policies/persistence.py @@ -0,0 +1,92 @@ +"""Transactional persistence for mutable user-policy associations.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, select + +from policyengine_api.data.v2.models import Policy, UserPolicy +from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.data.v2.user_policies.query import ( + UserPolicyRead, + association_read, + get_user_policy_row, +) +from policyengine_api.data.v2.user_policies.schemas import ( + UserPolicyCreateCommand, + UserPolicyPatchCommand, +) + + +class AssociationPolicyNotFoundError(LookupError): + """Raised when an association references an unknown policy UUID.""" + + +class AssociationCountryConflictError(ValueError): + """Raised when an association and its referenced policy differ by country.""" + + +def create_user_policy( + session: Session, + command: UserPolicyCreateCommand, +) -> UserPolicyRead: + """Create one independently identified association after policy validation.""" + + policy = session.exec( + select(Policy).where(Policy.id == command.policy_id) + ).one_or_none() + if policy is None: + raise AssociationPolicyNotFoundError( + f"policy {command.policy_id} was not found" + ) + if policy.country_id != command.country_id: + raise AssociationCountryConflictError( + "Association country_id must match the referenced policy" + ) + association = UserPolicy(**command.model_dump()) + session.add(association) + session.flush() + session.refresh(association) + return association_read(association) + + +def patch_user_policy( + session: Session, + *, + country_id: str, + association_id: UUID, + command: UserPolicyPatchCommand, +) -> UserPolicyRead: + """Change only explicitly supplied presentation fields.""" + + association = get_user_policy_row( + session, + country_id=country_id, + association_id=association_id, + ) + changes = command.model_dump(exclude_unset=True) + for field_name, value in changes.items(): + setattr(association, field_name, value) + association.updated_at = utc_now() + session.add(association) + session.flush() + session.refresh(association) + return association_read(association) + + +def delete_user_policy( + session: Session, + *, + country_id: str, + association_id: UUID, +) -> None: + """Delete one association; database cascades remove only its legacy mapping.""" + + association = get_user_policy_row( + session, + country_id=country_id, + association_id=association_id, + ) + session.delete(association) + session.flush() diff --git a/policyengine_api/data/v2/user_policies/query.py b/policyengine_api/data/v2/user_policies/query.py new file mode 100644 index 000000000..ec18e01f9 --- /dev/null +++ b/policyengine_api/data/v2/user_policies/query.py @@ -0,0 +1,115 @@ +"""Country-scoped v2 user-policy association reads.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from uuid import UUID + +from sqlmodel import Session, select + +from policyengine_api.data.v2.models import UserPolicy + + +class UserPolicyNotFoundError(LookupError): + """Raised when an association is absent from the selected country.""" + + +@dataclass(frozen=True) +class UserPolicyRead: + id: UUID + country_id: str + user_id: str + policy_id: UUID + name: str | None + description: str | None + created_at: datetime + updated_at: datetime + + +@dataclass(frozen=True) +class UserPolicyPage: + items: tuple[UserPolicyRead, ...] + offset: int + limit: int + has_more: bool + + +def association_read(association: UserPolicy) -> UserPolicyRead: + return UserPolicyRead( + id=association.id, + country_id=association.country_id, + user_id=association.user_id, + policy_id=association.policy_id, + name=association.name, + description=association.description, + created_at=association.created_at, + updated_at=association.updated_at, + ) + + +def get_user_policy_row( + session: Session, + *, + country_id: str, + association_id: UUID, +) -> UserPolicy: + association = session.exec( + select(UserPolicy).where( + UserPolicy.id == association_id, + UserPolicy.country_id == country_id, + ) + ).one_or_none() + if association is None: + raise UserPolicyNotFoundError( + f"user-policy association {association_id} was not found" + ) + return association + + +def read_user_policy( + session: Session, + *, + country_id: str, + association_id: UUID, +) -> UserPolicyRead: + """Read one association only under its stored country.""" + + return association_read( + get_user_policy_row( + session, + country_id=country_id, + association_id=association_id, + ) + ) + + +def list_user_policies( + session: Session, + *, + country_id: str, + user_id: str, + policy_id: UUID | None = None, + offset: int = 0, + limit: int = 100, +) -> UserPolicyPage: + """Read one deterministic bounded page for a supplied user identifier.""" + + statement = select(UserPolicy).where( + UserPolicy.country_id == country_id, + UserPolicy.user_id == user_id, + ) + if policy_id is not None: + statement = statement.where(UserPolicy.policy_id == policy_id) + rows = session.exec( + statement.order_by(UserPolicy.created_at, UserPolicy.id) + .offset(offset) + .limit(limit + 1) + ).all() + has_more = len(rows) > limit + return UserPolicyPage( + items=tuple(association_read(row) for row in rows[:limit]), + offset=offset, + limit=limit, + has_more=has_more, + ) diff --git a/policyengine_api/data/v2/user_policies/schemas.py b/policyengine_api/data/v2/user_policies/schemas.py new file mode 100644 index 000000000..933f0d2bd --- /dev/null +++ b/policyengine_api/data/v2/user_policies/schemas.py @@ -0,0 +1,34 @@ +"""Strict application commands for user-policy associations.""" + +from __future__ import annotations + +from typing import Annotated + +from pydantic import BaseModel, ConfigDict, StringConstraints, model_validator + +from policyengine_api.query_parameters import CountryId, ResourceId, UserId + + +class StrictAssociationCommand(BaseModel): + """Reject fields outside the reviewed association contract.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class UserPolicyCreateCommand(StrictAssociationCommand): + country_id: CountryId + user_id: UserId + policy_id: ResourceId + name: Annotated[str, StringConstraints(max_length=255)] | None = None + description: str | None = None + + +class UserPolicyPatchCommand(StrictAssociationCommand): + name: Annotated[str, StringConstraints(max_length=255)] | None = None + description: str | None = None + + @model_validator(mode="after") + def require_supplied_field(self) -> "UserPolicyPatchCommand": + if not self.model_fields_set.intersection({"name", "description"}): + raise ValueError("At least one of name or description must be supplied") + return self diff --git a/policyengine_api/data/v2/user_policies/service.py b/policyengine_api/data/v2/user_policies/service.py new file mode 100644 index 000000000..3562c7d01 --- /dev/null +++ b/policyengine_api/data/v2/user_policies/service.py @@ -0,0 +1,123 @@ +"""Session-owning application service for user-policy associations.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.data.v2.user_policies.persistence import ( + create_user_policy, + delete_user_policy, + patch_user_policy, +) +from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot +from policyengine_api.data.v2.user_policies.legacy import ( + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, + persist_legacy_user_policy, +) +from policyengine_api.data.v2.user_policies.query import ( + UserPolicyPage, + UserPolicyRead, + list_user_policies, + read_user_policy, +) +from policyengine_api.data.v2.user_policies.schemas import ( + UserPolicyCreateCommand, + UserPolicyPatchCommand, +) + + +class V2UserPolicyService: + """Own transaction boundaries for native association operations.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sessions = session_factory + + def create_user_policy( + self, + command: UserPolicyCreateCommand, + ) -> UserPolicyRead: + with self._sessions.begin() as session: + return create_user_policy(session, command) + + def get_user_policy( + self, + *, + country_id: str, + association_id: UUID, + ) -> UserPolicyRead: + with self._sessions() as session: + return read_user_policy( + session, + country_id=country_id, + association_id=association_id, + ) + + def list_user_policies( + self, + *, + country_id: str, + user_id: str, + policy_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> UserPolicyPage: + with self._sessions() as session: + return list_user_policies( + session, + country_id=country_id, + user_id=user_id, + policy_id=policy_id, + offset=offset, + limit=limit, + ) + + def patch_user_policy( + self, + *, + country_id: str, + association_id: UUID, + command: UserPolicyPatchCommand, + ) -> UserPolicyRead: + with self._sessions.begin() as session: + return patch_user_policy( + session, + country_id=country_id, + association_id=association_id, + command=command, + ) + + def delete_user_policy( + self, + *, + country_id: str, + association_id: UUID, + ) -> None: + with self._sessions.begin() as session: + delete_user_policy( + session, + country_id=country_id, + association_id=association_id, + ) + + def mirror_legacy_user_policy( + self, + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str], + ) -> LegacyUserPolicyPersistenceResult: + """Mirror one committed v1 saved policy in one Supabase transaction.""" + + with self._sessions.begin() as session: + return persist_legacy_user_policy( + session, + snapshot, + reform_snapshot, + source_revision=source_revision, + changed_fields=changed_fields, + ) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 4b3166689..3b7243b01 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -23,6 +23,30 @@ class V2MetadataResourceReader(Protocol): def close(self) -> None: ... +class V2PolicyResourceService(Protocol): + """Route-independent native policy operations for one request.""" + + def create_policy(self, command: object) -> object: ... + + def get_policy(self, *, country_id: str, policy_id: object) -> object: ... + + def list_policies(self, **filters: object) -> object: ... + + +class V2UserPolicyResourceService(Protocol): + """Route-independent native association operations for one request.""" + + def create_user_policy(self, command: object) -> object: ... + + def get_user_policy(self, **identity: object) -> object: ... + + def list_user_policies(self, **filters: object) -> object: ... + + def patch_user_policy(self, **changes: object) -> object: ... + + def delete_user_policy(self, **identity: object) -> None: ... + + class SimulationGatewayProbe(Protocol): """Minimal simulation-entrypoint health-check interface.""" @@ -68,6 +92,23 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: ) +def _default_v2_policy_service_factory() -> V2PolicyResourceService: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.data.v2.policies.service import V2PolicyService + + return V2PolicyService( + get_v2_session_factory(), + running_policyengine_version=_running_policyengine_version(), + ) + + +def _default_v2_user_policy_service_factory() -> V2UserPolicyResourceService: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.data.v2.user_policies.service import V2UserPolicyService + + return V2UserPolicyService(get_v2_session_factory()) + + @dataclass(frozen=True) class NativeRouteDependencies: """Runtime collaborators for native read routes.""" @@ -77,6 +118,10 @@ class NativeRouteDependencies: metadata_reader_factory: Callable[[], MetadataReader] specification_provider: Callable[[], JSONObject] v2_metadata_reader_factory: Callable[[], V2MetadataResourceReader] | None = None + v2_policy_service_factory: Callable[[], V2PolicyResourceService] | None = None + v2_user_policy_service_factory: Callable[[], V2UserPolicyResourceService] | None = ( + None + ) @classmethod def defaults(cls) -> "NativeRouteDependencies": @@ -87,4 +132,6 @@ def defaults(cls) -> "NativeRouteDependencies": metadata_reader_factory=_default_metadata_reader_factory, specification_provider=_default_specification_provider, v2_metadata_reader_factory=_default_v2_metadata_reader_factory, + v2_policy_service_factory=_default_v2_policy_service_factory, + v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), ) diff --git a/policyengine_api/fastapi_routes/query_parameters.py b/policyengine_api/fastapi_routes/query_parameters.py new file mode 100644 index 000000000..f8bd8a878 --- /dev/null +++ b/policyengine_api/fastapi_routes/query_parameters.py @@ -0,0 +1,66 @@ +"""FastAPI dependency adapter for canonical query-parameter models.""" + +from __future__ import annotations + +from collections.abc import Callable +from inspect import Parameter, Signature +from typing import Annotated, TypeVar + +from fastapi import Query, Request +from fastapi.exceptions import RequestValidationError + +from policyengine_api.query_parameters import ( + DuplicateScalarQueryParameterError, + StrictQueryParameters, + validate_scalar_query_multiplicity, +) + + +QueryParametersT = TypeVar("QueryParametersT", bound=StrictQueryParameters) + + +def _duplicate_error(error: DuplicateScalarQueryParameterError) -> dict[str, object]: + return { + "type": "value_error", + "loc": ("query", error.parameter_name), + "msg": "Input should occur only once for a scalar query parameter", + "input": None, + "ctx": {"error": error}, + } + + +def query_dependency( + model_type: type[QueryParametersT], +) -> Callable[..., QueryParametersT]: + """Build a typed dependency with runtime and OpenAPI query metadata.""" + + async def dependency( + request: Request, + query: QueryParametersT, + ) -> QueryParametersT: + try: + validate_scalar_query_multiplicity( + model_type, + request.query_params.multi_items(), + ) + except DuplicateScalarQueryParameterError as error: + raise RequestValidationError([_duplicate_error(error)]) from error + return query + + dependency.__name__ = f"parse_{model_type.__name__}" + dependency.__signature__ = Signature( # type: ignore[attr-defined] + parameters=( + Parameter( + "request", + kind=Parameter.POSITIONAL_OR_KEYWORD, + annotation=Request, + ), + Parameter( + "query", + kind=Parameter.POSITIONAL_OR_KEYWORD, + annotation=Annotated[model_type, Query()], + ), + ), + return_annotation=model_type, + ) + return dependency diff --git a/policyengine_api/fastapi_routes/v2_metadata.py b/policyengine_api/fastapi_routes/v2_metadata.py index 000eddcdd..b5a61fec4 100644 --- a/policyengine_api/fastapi_routes/v2_metadata.py +++ b/policyengine_api/fastapi_routes/v2_metadata.py @@ -16,6 +16,10 @@ from policyengine_api.fastapi_routes.v2_metadata_parameters import ( build_v2_metadata_parameter_router, ) +from policyengine_api.fastapi_routes.v2_policies import build_v2_policy_router +from policyengine_api.fastapi_routes.v2_user_policies import ( + build_v2_user_policy_router, +) def build_v2_metadata_router( @@ -24,6 +28,8 @@ def build_v2_metadata_router( """Build isolated resource routes without loading v2 configuration.""" router = APIRouter() + router.include_router(build_v2_user_policy_router(dependencies)) + router.include_router(build_v2_policy_router(dependencies)) router.include_router(build_v2_metadata_model_router(dependencies)) router.include_router(build_v2_metadata_parameter_router(dependencies)) router.include_router(build_v2_metadata_geography_router(dependencies)) diff --git a/policyengine_api/fastapi_routes/v2_policies.py b/policyengine_api/fastapi_routes/v2_policies.py new file mode 100644 index 000000000..f3efe57d3 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2_policies.py @@ -0,0 +1,197 @@ +"""Native FastAPI routes for immutable v2 policies.""" + +from __future__ import annotations + +from collections.abc import Callable +from uuid import UUID + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.exc import SQLAlchemyError +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.data.v2.policies.api_schemas import ( + MAXIMUM_POLICY_REQUEST_BYTES, + POLICY_ERROR_RESPONSES, + PolicyCreateRequest, + PolicyDetailResponse, + PolicyDetailResult, + PolicyErrorResponse, + PolicyItem, + PolicyPageResponse, + PolicyPageResult, +) +from policyengine_api.data.v2.policies.catalog import PolicyCatalogValidationError +from policyengine_api.data.v2.policies.persistence import ( + PolicyContentHashCollisionError, + PolicyPersistenceIntegrityError, +) +from policyengine_api.data.v2.policies.query import PolicyNotFoundError +from policyengine_api.data.v2.policies.schemas import NativePolicyCreateCommand +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2PolicyResourceService, +) +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + PolicyCollectionQuery, + PolicyCreateQuery, + PolicyDetailQuery, +) + + +class PolicyRequestTooLargeError(ValueError): + """Raised before persistence when a native policy body exceeds 1 MiB.""" + + +async def enforce_policy_request_size(request: Request) -> None: + """Bound both declared and actual request bytes before service creation.""" + + content_length = request.headers.get("content-length") + if content_length is not None: + try: + declared_length = int(content_length) + except ValueError as error: + raise PolicyRequestTooLargeError( + "Policy request Content-Length is invalid" + ) from error + if declared_length > MAXIMUM_POLICY_REQUEST_BYTES: + raise PolicyRequestTooLargeError("Policy request body exceeds 1 MiB") + if len(await request.body()) > MAXIMUM_POLICY_REQUEST_BYTES: + raise PolicyRequestTooLargeError("Policy request body exceeds 1 MiB") + + +def policy_error_response(status_code: int, message: str) -> JSONResponse: + error = PolicyErrorResponse(message=message) + return JSONResponse( + status_code=status_code, + content=error.model_dump(mode="json"), + ) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2PolicyResourceService]: + if dependencies.v2_policy_service_factory is not None: + return dependencies.v2_policy_service_factory + from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_policy_service_factory, + ) + + return _default_v2_policy_service_factory + + +def _policy_operation(operation: Callable[[], object]) -> object | JSONResponse: + try: + return operation() + except PolicyCatalogValidationError as error: + return policy_error_response(400, str(error)) + except (MetadataCatalogVersionNotFoundError, PolicyNotFoundError) as error: + return policy_error_response(404, str(error)) + except PolicyContentHashCollisionError: + return policy_error_response(409, "Policy content hash conflicts with storage") + except PolicyPersistenceIntegrityError: + return policy_error_response(500, "Stored policy integrity failed") + except (V2ConfigurationError, MetadataCatalogUnavailableError, SQLAlchemyError): + return policy_error_response(503, "V2 policy persistence is unavailable") + except Exception: # noqa: BLE001 - route must return a secret-safe typed error + return policy_error_response(500, "V2 policy operation failed") + + +def build_v2_policy_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build native policy routes without opening a database connection.""" + + router = APIRouter(prefix="/v2", responses=POLICY_ERROR_RESPONSES) + create_query = query_dependency(PolicyCreateQuery) + detail_query = query_dependency(PolicyDetailQuery) + collection_query = query_dependency(PolicyCollectionQuery) + service_factory = _service_factory(dependencies) + + @router.post( + "/policies", + response_model=PolicyDetailResponse, + status_code=201, + responses={ + 200: { + "model": PolicyDetailResponse, + "description": "Equivalent immutable content already exists.", + } + }, + summary="Create or find an immutable policy", + ) + def create_policy( + body: PolicyCreateRequest, + query: PolicyCreateQuery = Depends(create_query), + _size: None = Depends(enforce_policy_request_size), + ) -> PolicyDetailResponse | JSONResponse: + if body.country_id != query.country_id: + return policy_error_response( + 400, + "Body country_id must match query country_id", + ) + + def create() -> PolicyDetailResponse | JSONResponse: + result = service_factory().create_policy( + NativePolicyCreateCommand( + **body.model_dump(), + policyengine_version=query.policyengine_version, + ) + ) + response = PolicyDetailResponse( + result=PolicyDetailResult(item=PolicyItem.from_read(result.item)) + ) + if result.created: + return response + return JSONResponse( + status_code=200, + content=response.model_dump(mode="json"), + ) + + return _policy_operation(create) + + @router.get( + "/policies/{policy_id}", + response_model=PolicyDetailResponse, + summary="Read one immutable policy", + ) + def get_policy( + policy_id: UUID, + query: PolicyDetailQuery = Depends(detail_query), + ) -> PolicyDetailResponse | JSONResponse: + def read() -> PolicyDetailResponse: + item = service_factory().get_policy( + country_id=query.country_id, + policy_id=policy_id, + ) + return PolicyDetailResponse( + result=PolicyDetailResult(item=PolicyItem.from_read(item)) + ) + + return _policy_operation(read) + + @router.get( + "/policies", + response_model=PolicyPageResponse, + summary="List immutable policies", + ) + def get_policies( + query: PolicyCollectionQuery = Depends(collection_query), + ) -> PolicyPageResponse | JSONResponse: + def read() -> PolicyPageResponse: + page = service_factory().list_policies( + country_id=query.country_id, + tax_benefit_model_id=query.tax_benefit_model_id, + offset=query.offset, + limit=query.limit, + ) + return PolicyPageResponse(result=PolicyPageResult.from_page(page)) + + return _policy_operation(read) + + return router diff --git a/policyengine_api/fastapi_routes/v2_user_policies.py b/policyengine_api/fastapi_routes/v2_user_policies.py new file mode 100644 index 000000000..7f2f9c4f3 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2_user_policies.py @@ -0,0 +1,203 @@ +"""Native FastAPI routes for mutable v2 user-policy associations.""" + +from __future__ import annotations + +from collections.abc import Callable +from uuid import UUID + +from fastapi import APIRouter, Depends +from sqlalchemy.exc import SQLAlchemyError +from starlette.responses import JSONResponse, Response + +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.data.v2.user_policies.api_schemas import ( + USER_POLICY_ERROR_RESPONSES, + UserPolicyCreateRequest, + UserPolicyDetailResponse, + UserPolicyDetailResult, + UserPolicyErrorResponse, + UserPolicyItem, + UserPolicyPageResponse, + UserPolicyPageResult, + UserPolicyPatchRequest, +) +from policyengine_api.data.v2.user_policies.persistence import ( + AssociationCountryConflictError, + AssociationPolicyNotFoundError, +) +from policyengine_api.data.v2.user_policies.query import UserPolicyNotFoundError +from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2UserPolicyResourceService, +) +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + CountryQuery, + UserPolicyCollectionQuery, +) + + +def user_policy_error_response(status_code: int, message: str) -> JSONResponse: + error = UserPolicyErrorResponse(message=message) + return JSONResponse( + status_code=status_code, + content=error.model_dump(mode="json"), + ) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2UserPolicyResourceService]: + if dependencies.v2_user_policy_service_factory is not None: + return dependencies.v2_user_policy_service_factory + from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_user_policy_service_factory, + ) + + return _default_v2_user_policy_service_factory + + +def _association_operation( + operation: Callable[[], object], +) -> object | JSONResponse: + try: + return operation() + except AssociationCountryConflictError as error: + return user_policy_error_response(400, str(error)) + except (AssociationPolicyNotFoundError, UserPolicyNotFoundError) as error: + return user_policy_error_response(404, str(error)) + except (V2ConfigurationError, SQLAlchemyError): + return user_policy_error_response( + 503, + "V2 association persistence is unavailable", + ) + except Exception: # noqa: BLE001 - return a secret-safe typed error + return user_policy_error_response(500, "V2 association operation failed") + + +def build_v2_user_policy_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build native association routes without opening a database connection.""" + + router = APIRouter(prefix="/v2", responses=USER_POLICY_ERROR_RESPONSES) + country_query = query_dependency(CountryQuery) + collection_query = query_dependency(UserPolicyCollectionQuery) + service_factory = _service_factory(dependencies) + + @router.post( + "/user-policies", + response_model=UserPolicyDetailResponse, + status_code=201, + summary="Create a user-policy association", + description=( + "Creates a saved association for an unverified caller-supplied " + "user identifier. This operation performs no authentication or " + "authorization check." + ), + ) + def create_user_policy( + body: UserPolicyCreateRequest, + query: CountryQuery = Depends(country_query), + ) -> UserPolicyDetailResponse | JSONResponse: + if body.country_id != query.country_id: + return user_policy_error_response( + 400, + "Body country_id must match query country_id", + ) + + def create() -> UserPolicyDetailResponse: + item = service_factory().create_user_policy(body) + return UserPolicyDetailResponse( + result=UserPolicyDetailResult(item=UserPolicyItem.from_read(item)) + ) + + return _association_operation(create) + + @router.get( + "/user-policies/{association_id}", + response_model=UserPolicyDetailResponse, + summary="Read one user-policy association", + ) + def get_user_policy( + association_id: UUID, + query: CountryQuery = Depends(country_query), + ) -> UserPolicyDetailResponse | JSONResponse: + def read() -> UserPolicyDetailResponse: + item = service_factory().get_user_policy( + country_id=query.country_id, + association_id=association_id, + ) + return UserPolicyDetailResponse( + result=UserPolicyDetailResult(item=UserPolicyItem.from_read(item)) + ) + + return _association_operation(read) + + @router.get( + "/user-policies", + response_model=UserPolicyPageResponse, + summary="List user-policy associations", + description=( + "Filters by an unverified caller-supplied user identifier. A match " + "is not an authentication or authorization decision." + ), + ) + def get_user_policies( + query: UserPolicyCollectionQuery = Depends(collection_query), + ) -> UserPolicyPageResponse | JSONResponse: + def read() -> UserPolicyPageResponse: + page = service_factory().list_user_policies( + country_id=query.country_id, + user_id=query.user_id, + policy_id=query.policy_id, + offset=query.offset, + limit=query.limit, + ) + return UserPolicyPageResponse(result=UserPolicyPageResult.from_page(page)) + + return _association_operation(read) + + @router.patch( + "/user-policies/{association_id}", + response_model=UserPolicyDetailResponse, + summary="Update association presentation fields", + ) + def patch_user_policy_route( + association_id: UUID, + body: UserPolicyPatchRequest, + query: CountryQuery = Depends(country_query), + ) -> UserPolicyDetailResponse | JSONResponse: + def patch() -> UserPolicyDetailResponse: + item = service_factory().patch_user_policy( + country_id=query.country_id, + association_id=association_id, + command=body, + ) + return UserPolicyDetailResponse( + result=UserPolicyDetailResult(item=UserPolicyItem.from_read(item)) + ) + + return _association_operation(patch) + + @router.delete( + "/user-policies/{association_id}", + response_model=None, + status_code=204, + response_class=Response, + summary="Delete one user-policy association", + ) + def delete_user_policy_route( + association_id: UUID, + query: CountryQuery = Depends(country_query), + ) -> Response | JSONResponse: + def delete() -> Response: + service_factory().delete_user_policy( + country_id=query.country_id, + association_id=association_id, + ) + return Response(status_code=204) + + return _association_operation(delete) + + return router diff --git a/policyengine_api/migration_flags.py b/policyengine_api/migration_flags.py index 5cd95f20a..0d4cb44c2 100644 --- a/policyengine_api/migration_flags.py +++ b/policyengine_api/migration_flags.py @@ -30,6 +30,8 @@ class RouteImplementation(StrEnum): ) DB_WRITE_SOURCES = frozenset({"cloud_sql", "dual_write", "supabase"}) DB_READ_SOURCES = frozenset({"cloud_sql", "read_compare", "supabase"}) +V1_POLICY_WRITE_SOURCES = frozenset({"cloud_sql", "dual_write"}) +V1_POLICY_READ_SOURCES = frozenset({"cloud_sql"}) SIM_ENTRYPOINTS = frozenset({"old_gateway_direct", "cloud_run_simulation_entrypoint"}) SIM_COMPUTE_BACKENDS = frozenset( {"old_gateway", "v2_shadow", "v2_percent", "v2_primary"} @@ -143,6 +145,26 @@ def get_db_read(entity: str) -> str: return _read_choice(env_name, DEFAULT_DB_SOURCE, DB_READ_SOURCES) +def get_v1_policy_write_source() -> str: + """Select Cloud SQL alone or immediate Cloud SQL-to-Supabase mirroring.""" + + return _read_choice( + "DB_WRITE_POLICY", + DEFAULT_DB_SOURCE, + V1_POLICY_WRITE_SOURCES, + ) + + +def get_v1_policy_read_source() -> str: + """Require every v1 policy read to remain on Cloud SQL in Phase 10.""" + + return _read_choice( + "DB_READ_POLICY", + DEFAULT_DB_SOURCE, + V1_POLICY_READ_SOURCES, + ) + + def get_sim_compute(flow: str) -> str: env_name = f"SIM_COMPUTE_{flow.upper()}" return _read_choice( diff --git a/policyengine_api/migration_logging.py b/policyengine_api/migration_logging.py index 3de5ddfcb..701ea0886 100644 --- a/policyengine_api/migration_logging.py +++ b/policyengine_api/migration_logging.py @@ -29,6 +29,7 @@ "variables", } ) +V2_POLICY_RESOURCE_SEGMENTS = frozenset({"policies", "user-policies"}) def _is_v2_metadata_resource_read(method: str, path: str) -> bool: @@ -42,6 +43,17 @@ def _is_v2_metadata_resource_read(method: str, path: str) -> bool: ) +def _is_v2_policy_resource(method: str, path: str) -> bool: + if method not in {"GET", "POST", "PATCH", "DELETE"}: + return False + segments = [segment for segment in path.strip("/").split("/") if segment] + return ( + len(segments) >= 2 + and segments[0] == "v2" + and segments[1] in V2_POLICY_RESOURCE_SEGMENTS + ) + + def register_migration_request_logging(app: flask.Flask) -> None: """Register request IDs and migration logging for Flask.""" @@ -95,11 +107,22 @@ def log_migration_request( route_group = infer_route_group(path) is_v2_metadata_read = _is_v2_metadata_resource_read(method, path) + is_v2_policy_resource = _is_v2_policy_resource(method, path) + uses_explicit_v2_source = is_v2_metadata_read or is_v2_policy_resource migration_context = get_migration_log_context( route_group, route_impl=route_impl, - use_configured_db_sources=not is_v2_metadata_read, - db_read_source="supabase" if is_v2_metadata_read else None, + use_configured_db_sources=not uses_explicit_v2_source, + db_write_source=( + "supabase" + if is_v2_policy_resource and method in {"POST", "PATCH", "DELETE"} + else None + ), + db_read_source=( + "supabase" + if is_v2_metadata_read or (is_v2_policy_resource and method == "GET") + else None + ), ) logger.log_struct( diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index a19376b8c..f7403f58b 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -44,7 +44,7 @@ class RouteGroupConfig: ), RouteGroupConfig( name="policy", - path_segments=("policy", "policies", "user-policy"), + path_segments=("policy", "policies", "user-policy", "user-policies"), db_entity="policy", ), RouteGroupConfig( diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py new file mode 100644 index 000000000..a889b79f7 --- /dev/null +++ b/policyengine_api/query_parameters.py @@ -0,0 +1,203 @@ +"""Canonical query-parameter definitions shared by HTTP frameworks.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from types import UnionType +from typing import Annotated, Any, Literal, TypeVar, Union, get_args, get_origin +from uuid import UUID + +from pydantic import ( + AfterValidator, + BaseModel, + BeforeValidator, + ConfigDict, + Field, +) + +from policyengine_api.data.v2.catalog.catalog_selection import ( + validate_policyengine_version, +) + + +DEFAULT_QUERY_LIMIT = 100 +MAXIMUM_QUERY_LIMIT = 500 +MAXIMUM_USER_ID_LENGTH = 255 +SUPPORTED_COUNTRY_IDS = frozenset({"us", "uk"}) + + +def normalize_country_id(value: Any) -> Any: + """Lowercase a textual country ID before its supported-value check.""" + + return value.lower() if isinstance(value, str) else value + + +def validate_user_id(value: str) -> str: + """Reject an empty or whitespace-only caller-supplied identifier.""" + + if not value.strip(): + raise ValueError("user_id must contain at least one non-whitespace character") + return value + + +CountryId = Annotated[ + Literal["us", "uk"], + BeforeValidator(normalize_country_id), + Field(description="Supported PolicyEngine country ID"), +] +PolicyEngineVersion = Annotated[ + str, + Field(max_length=128, description="Canonical non-placeholder PEP 440 version"), + AfterValidator(validate_policyengine_version), +] +Offset = Annotated[int, Field(ge=0, description="Zero-based result offset")] +Limit = Annotated[ + int, + Field( + ge=1, + le=MAXIMUM_QUERY_LIMIT, + description="Maximum number of returned resources", + ), +] +ResourceId = Annotated[UUID, Field(description="Exact resource UUID")] +UserId = Annotated[ + str, + Field( + min_length=1, + max_length=MAXIMUM_USER_ID_LENGTH, + description="Unverified caller-supplied user identifier", + ), + AfterValidator(validate_user_id), +] + + +class StrictQueryParameters(BaseModel): + """Base for query contracts that reject every undeclared field.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class CountryQuery(StrictQueryParameters): + """Required country selection shared by country-scoped routes.""" + + country_id: CountryId + + +class CatalogQuery(CountryQuery): + """Country plus an optional exact PolicyEngine.py catalog version.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class PaginationQuery(StrictQueryParameters): + """Canonical bounded offset/limit pagination.""" + + offset: Offset = 0 + limit: Limit = DEFAULT_QUERY_LIMIT + + +class PolicyCreateQuery(CatalogQuery): + """Query contract for native immutable policy creation.""" + + +class PolicyDetailQuery(CountryQuery): + """Query contract for country-scoped policy detail reads.""" + + +class PolicyCollectionQuery(CountryQuery, PaginationQuery): + """Query contract for an exact-filtered policy collection.""" + + tax_benefit_model_id: ResourceId | None = None + + +class UserPolicyCollectionQuery(CountryQuery, PaginationQuery): + """Query contract for one caller-supplied user's policy associations.""" + + user_id: UserId + policy_id: ResourceId | None = None + + +QueryParametersT = TypeVar("QueryParametersT", bound=StrictQueryParameters) + + +class DuplicateScalarQueryParameterError(ValueError): + """Raised when a scalar query field is supplied more than once.""" + + def __init__(self, parameter_name: str) -> None: + self.parameter_name = parameter_name + super().__init__( + f"scalar query parameter {parameter_name!r} must not be repeated" + ) + + +def _annotation_is_list(annotation: object) -> bool: + origin = get_origin(annotation) + if origin is list: + return True + if origin in (Union, UnionType): + return any(_annotation_is_list(member) for member in get_args(annotation)) + return False + + +def query_field_multiplicity( + model_type: type[StrictQueryParameters], +) -> Mapping[str, bool]: + """Return public query names mapped to whether repeated values are valid.""" + + return { + (field.alias or field_name): _annotation_is_list(field.annotation) + for field_name, field in model_type.model_fields.items() + } + + +def validate_scalar_query_multiplicity( + model_type: type[StrictQueryParameters], + items: Iterable[tuple[str, str]], +) -> None: + """Reject a repeated declared scalar while allowing declared list fields.""" + + multiplicity = query_field_multiplicity(model_type) + seen: set[str] = set() + for name, _value in items: + if name not in multiplicity or multiplicity[name]: + continue + if name in seen: + raise DuplicateScalarQueryParameterError(name) + seen.add(name) + + +def parse_query_items( + model_type: type[QueryParametersT], + items: Iterable[tuple[str, str]], +) -> QueryParametersT: + """Validate ordered query pairs through one canonical Pydantic schema.""" + + pairs = list(items) + validate_scalar_query_multiplicity(model_type, pairs) + multiplicity = query_field_multiplicity(model_type) + values: dict[str, object] = {} + for name, value in pairs: + if multiplicity.get(name, False): + values.setdefault(name, []) + list_values = values[name] + if isinstance(list_values, list): + list_values.append(value) + continue + values[name] = value + return model_type.model_validate(values) + + +def parse_multidict_query( + model_type: type[QueryParametersT], + query_input: object, +) -> QueryParametersT: + """Adapt a Flask/Werkzeug-style MultiDict to the canonical parser.""" + + items_method = getattr(query_input, "items", None) + if not callable(items_method): + raise TypeError("query input must provide an items method") + try: + items = items_method(multi=True) + except TypeError as error: + raise TypeError("query input must preserve repeated keys") from error + return parse_query_items(model_type, items) diff --git a/policyengine_api/readiness.py b/policyengine_api/readiness.py index 2ad374729..353df28e3 100644 --- a/policyengine_api/readiness.py +++ b/policyengine_api/readiness.py @@ -13,6 +13,28 @@ _ready = True +def validate_policy_runtime_configuration() -> None: + """Validate Phase 10 policy sources and conditionally require Supabase.""" + + from policyengine_api.data.v2.settings import ( + load_v2_runtime_database_settings, + ) + from policyengine_api.migration_flags import ( + RouteImplementation, + get_route_impl, + get_v1_policy_read_source, + get_v1_policy_write_source, + ) + + write_source = get_v1_policy_write_source() + get_v1_policy_read_source() + native_policy_routes = ( + get_route_impl("policy") is RouteImplementation.FASTAPI_NATIVE + ) + if write_source == "dual_write" or native_policy_routes: + load_v2_runtime_database_settings() + + def mark_not_ready() -> None: """Report not-ready — call before running the startup warmup.""" global _ready @@ -30,4 +52,11 @@ def mark_ready() -> None: def is_ready() -> bool: """Whether the service is warmed up and can serve a real request quickly.""" with _lock: - return _ready + warmed_up = _ready + if not warmed_up: + return False + try: + validate_policy_runtime_configuration() + except (RuntimeError, ValueError): + return False + return True diff --git a/policyengine_api/routes/policy_routes.py b/policyengine_api/routes/policy_routes.py index f857954a4..e3bfe3a7a 100644 --- a/policyengine_api/routes/policy_routes.py +++ b/policyengine_api/routes/policy_routes.py @@ -1,12 +1,31 @@ import json +import time from flask import Blueprint, Response, request from werkzeug.exceptions import BadRequest, NotFound from policyengine_api.data.v1_models import Policy, UserPolicy +from policyengine_api.gcp_logging import logger +from policyengine_api.migration_flags import ( + get_v1_policy_read_source, + get_v1_policy_write_source, +) +from policyengine_api.request_context import current_request_id from policyengine_api.response_factory import _make_error_response +from policyengine_api.services.policy_mirroring import ( + PolicyMirrorUnavailableError, + mirror_policy_after_commit, +) from policyengine_api.services.policy_service import PolicyService -from policyengine_api.services.user_policy_service import UserPolicyService +from policyengine_api.services.user_policy_service import ( + USER_POLICY_MUTABLE_FIELDS, + UserPolicyPersistenceError, + UserPolicyService, +) +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorUnavailableError, + mirror_pending_user_policy_events_after_commit, +) from policyengine_api.utils.payload_validators import ( validate_country, validate_set_policy_payload, @@ -17,6 +36,91 @@ user_policy_service = UserPolicyService() +def _policy_configuration_unavailable() -> Response: + return _make_error_response( + "Policy persistence configuration is unavailable.", + 503, + ) + + +def _policy_mirror_unavailable() -> Response: + return _make_error_response( + "V2 policy mirroring is unavailable; retry the same request.", + 503, + ) + + +def _user_policy_mirror_unavailable() -> Response: + return _make_error_response( + "V2 saved-policy mirroring is unavailable; retry the same request.", + 503, + include_status=False, + ) + + +def _safe_legacy_user_policy_id(value: object) -> int | None: + if ( + isinstance(value, int) + and not isinstance(value, bool) + and 0 <= value <= 2_147_483_647 + ): + return value + return None + + +def _user_policy_persistence_failure( + error: UserPolicyPersistenceError, + *, + operation: str, + country_id: str, + configured_write_source: str, + started_at: float, + legacy_user_policy_id: object | None = None, +) -> Response: + """Return and record a failure without serializing exception details.""" + + status_code = 503 if error.retryable else 500 + try: + logger.log_struct( + { + "message": "V1 saved-policy persistence failed", + "metric_name": "v1_user_policy_persistence_failures", + "metric_value": 1, + "resource": "user_policy", + "operation": operation, + "database_source": "cloud_sql", + "configured_write_source": configured_write_source, + "country_id": country_id, + "legacy_user_policy_id": _safe_legacy_user_policy_id( + legacy_user_policy_id + ), + "request_id": current_request_id(), + "outcome": "error", + "failure_category": error.category.value, + "http_status": status_code, + "duration_ms": round( + (time.perf_counter() - started_at) * 1000, + 3, + ), + }, + severity="ERROR", + ) + except Exception: + # A logging failure must not replace the persistence response. + pass + + message = ( + "Policy database is temporarily unavailable; please try again later." + if status_code == 503 + else "Internal database error; please try again later." + ) + return _make_error_response( + message, + status_code, + include_status=False, + ) + + def _serialize_policy(policy: Policy) -> dict: return { "id": policy.id, @@ -43,6 +147,11 @@ def get_policy(country_id: str, policy_id: int | str) -> Response: policy data in JSON format """ + try: + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() + # Specifically cast policy_id to an integer policy_id = int(policy_id) @@ -75,14 +184,29 @@ def set_policy(country_id: str) -> Response: if not is_payload_valid: raise BadRequest(f"Invalid JSON data; details: {message}") + try: + write_source = get_v1_policy_write_source() + except ValueError: + return _policy_configuration_unavailable() + label = payload.pop("label", None) policy_json = payload.pop("data", None) - policy_id, message, is_existing_policy = policy_service.set_policy( + creation = policy_service.set_policy( country_id, label, policy_json, ) + policy_id, message, is_existing_policy = creation + + if write_source == "dual_write": + snapshot = getattr(creation, "snapshot", None) + if snapshot is None: + return _policy_mirror_unavailable() + try: + mirror_policy_after_commit(snapshot) + except PolicyMirrorUnavailableError: + return _policy_mirror_unavailable() response_body = dict( status="ok", @@ -100,6 +224,7 @@ def _serialize_user_policy(user_policy: UserPolicy) -> dict: return { column.name: getattr(user_policy, column.name) for column in UserPolicy.__table__.columns + if column.name != "mirror_revision" } @@ -110,6 +235,11 @@ def get_policy_search(country_id: str) -> Response: query = request.args.get("query", "") unique_only = request.args.get("unique_only", default=False, type=json.loads) + try: + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() + try: results = policy_service.search_policies( country_id, @@ -134,9 +264,9 @@ def get_policy_search(country_id: str) -> Response: status=200, mimetype="application/json", ) - except Exception as error: + except Exception: return _make_error_response( - f"Internal server error: {error}", + "Internal server error; please try again later.", 500, ) @@ -180,28 +310,58 @@ def set_user_policy(country_id: str) -> Response: } try: - creation = user_policy_service.create_or_get_user_policy(values) + write_source = get_v1_policy_write_source() + if write_source == "dual_write": + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() + + persistence_started_at = time.perf_counter() + try: + creation = user_policy_service.create_or_get_user_policy( + values, + record_mirror_event=write_source == "dual_write", + ) user_policy = creation.user_policy - if not creation.created: - return Response( - json.dumps( - dict( - status="ok", - message=( - f"The reform #{reform_id} / baseline #{baseline_id} pair " - f"already exists for user {user_id}" - ), - result=dict(id=user_policy.id), - ) - ), - status=200, - mimetype="application/json", + except UserPolicyPersistenceError as error: + return _user_policy_persistence_failure( + error, + operation="create", + country_id=country_id, + configured_write_source=write_source, + started_at=persistence_started_at, + ) + + if write_source == "dual_write": + if creation.mirror_revision is None: + return _user_policy_mirror_unavailable() + try: + mirror_pending_user_policy_events_after_commit( + country_id, + creation.user_policy.id, + through_revision=creation.mirror_revision, + event_service=user_policy_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, ) - except Exception as error: - return _make_error_response( - f"Internal database error: {error}; please try again later.", - 500, - include_status=False, + except UserPolicyMirrorUnavailableError: + return _user_policy_mirror_unavailable() + except Exception: + return _user_policy_mirror_unavailable() + + if not creation.created: + return Response( + json.dumps( + dict( + status="ok", + message=( + f"The reform #{reform_id} / baseline #{baseline_id} pair " + f"already exists for user {user_id}" + ), + result=dict(id=user_policy.id), + ) + ), + status=200, + mimetype="application/json", ) return Response( @@ -219,8 +379,12 @@ def set_user_policy(country_id: str) -> Response: @policy_bp.route("//user-policy/", methods=["GET"]) @validate_country -def get_user_policy(country_id: str, user_id: str) -> dict: +def get_user_policy(country_id: str, user_id: str) -> dict | Response: """Fetch all saved policies for a user.""" + try: + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() user_policies = user_policy_service.list_user_policies(country_id, user_id) return dict( status="ok", @@ -229,21 +393,7 @@ def get_user_policy(country_id: str, user_id: str) -> dict: ) -UPDATE_USER_POLICY_ALLOWED_FIELDS = frozenset( - { - "reform_label", - "baseline_label", - "year", - "geography", - "dataset", - "number_of_provisions", - "api_version", - "added_date", - "updated_date", - "budgetary_impact", - "type", - } -) +UPDATE_USER_POLICY_ALLOWED_FIELDS = USER_POLICY_MUTABLE_FIELDS @policy_bp.route("//user-policy", methods=["PUT"]) @@ -277,25 +427,53 @@ def update_user_policy(country_id: str) -> Response: ) try: - user_policy = user_policy_service.update_user_policy( + write_source = get_v1_policy_write_source() + if write_source == "dual_write": + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() + + persistence_started_at = time.perf_counter() + try: + update = user_policy_service.update_user_policy( country_id, user_policy_id, payload, + record_mirror_event=write_source == "dual_write", ) - except Exception as error: - return _make_error_response( - f"Internal database error: {error}; please try again later.", - 500, - include_status=False, + except UserPolicyPersistenceError as error: + return _user_policy_persistence_failure( + error, + operation="update", + country_id=country_id, + configured_write_source=write_source, + started_at=persistence_started_at, + legacy_user_policy_id=user_policy_id, ) - if user_policy is None: + if update is None: return _make_error_response( f"User policy #{user_policy_id} not found.", 404, include_status=False, ) + if write_source == "dual_write": + if update.mirror_revision is None: + return _user_policy_mirror_unavailable() + try: + mirror_pending_user_policy_events_after_commit( + country_id, + update.user_policy.id, + through_revision=update.mirror_revision, + event_service=user_policy_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + ) + except UserPolicyMirrorUnavailableError: + return _user_policy_mirror_unavailable() + except Exception: + return _user_policy_mirror_unavailable() + return Response( json.dumps( dict( diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py new file mode 100644 index 000000000..aa1a3449f --- /dev/null +++ b/policyengine_api/services/policy_mirroring.py @@ -0,0 +1,146 @@ +"""Immediate, observable mirroring of committed v1 policies into v2.""" + +from __future__ import annotations + +from collections.abc import Callable +import time +from typing import Protocol + +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.data.v2.policies.catalog import PolicyCatalogValidationError +from policyengine_api.data.v2.policies.legacy import ( + LegacyPolicyMappingIntegrityError, + LegacyPolicyPersistenceResult, + LegacyPolicySnapshot, + LegacyPolicyTranslationError, +) +from policyengine_api.data.v2.policies.persistence import ( + PolicyContentHashCollisionError, + PolicyPersistenceIntegrityError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.gcp_logging import logger + + +class LegacyPolicyMirror(Protocol): + """Supabase transaction service used after a v1 commit.""" + + def mirror_legacy_policy( + self, + snapshot: LegacyPolicySnapshot, + ) -> LegacyPolicyPersistenceResult: ... + + +class PolicyMirrorUnavailableError(RuntimeError): + """Raised when a committed v1 policy could not be mirrored immediately.""" + + +def _default_mirror_factory() -> LegacyPolicyMirror: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.data.v2.policies.service import V2PolicyService + + return V2PolicyService(get_v2_session_factory()) + + +def _failure_category(error: Exception) -> str: + if isinstance(error, V2ConfigurationError): + return "configuration" + if isinstance( + error, + ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, + PolicyCatalogValidationError, + LegacyPolicyTranslationError, + ), + ): + return "catalog_or_translation" + if isinstance( + error, + ( + LegacyPolicyMappingIntegrityError, + PolicyContentHashCollisionError, + PolicyPersistenceIntegrityError, + ), + ): + return "integrity" + if isinstance(error, SQLAlchemyError): + return "database" + return "unexpected" + + +def _log_mirror_event( + *, + snapshot: LegacyPolicySnapshot, + started_at: float, + outcome: str, + actual_write_sources: list[str], + destination_policy_id: object | None = None, + failure_category: str | None = None, + policy_created: bool | None = None, + mapping_created: bool | None = None, +) -> None: + logger.log_struct( + { + "message": "V1 policy immediate mirror completed", + "metric_name": "v1_policy_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": actual_write_sources, + "country_id": snapshot.country_id, + "legacy_policy_id": snapshot.legacy_policy_id, + "destination_policy_id": ( + str(destination_policy_id) + if destination_policy_id is not None + else None + ), + "outcome": outcome, + "failure_category": failure_category, + "policy_created": policy_created, + "mapping_created": mapping_created, + "duration_ms": round((time.perf_counter() - started_at) * 1000, 3), + }, + severity="INFO" if outcome == "ok" else "ERROR", + ) + + +def mirror_policy_after_commit( + snapshot: LegacyPolicySnapshot, + *, + mirror_factory: Callable[[], LegacyPolicyMirror] | None = None, +) -> LegacyPolicyPersistenceResult: + """Require the Supabase policy and mapping transaction before v1 success.""" + + started_at = time.perf_counter() + try: + result = (mirror_factory or _default_mirror_factory)().mirror_legacy_policy( + snapshot + ) + except Exception as error: + _log_mirror_event( + snapshot=snapshot, + started_at=started_at, + outcome="error", + actual_write_sources=["cloud_sql"], + failure_category=_failure_category(error), + ) + raise PolicyMirrorUnavailableError( + "The committed v1 policy could not be mirrored to v2" + ) from error + + _log_mirror_event( + snapshot=snapshot, + started_at=started_at, + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + destination_policy_id=result.policy_id, + policy_created=result.policy_created, + mapping_created=result.mapping_created, + ) + return result diff --git a/policyengine_api/services/policy_service.py b/policyengine_api/services/policy_service.py index 728565882..ad6b36f14 100644 --- a/policyengine_api/services/policy_service.py +++ b/policyengine_api/services/policy_service.py @@ -1,5 +1,8 @@ from __future__ import annotations +import copy +from dataclasses import dataclass +from collections.abc import Iterator from typing import Any from sqlalchemy import select @@ -8,9 +11,27 @@ from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import Policy +from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot from policyengine_api.utils import hash_object +@dataclass(frozen=True) +class PolicySetResult: + """Existing v1 return values plus a detached committed-row snapshot.""" + + policy_id: int + message: str + is_existing_policy: bool + snapshot: LegacyPolicySnapshot + + def __iter__(self) -> Iterator[int | str | bool]: + """Preserve the established three-value internal unpacking interface.""" + + yield self.policy_id + yield self.message + yield self.is_existing_policy + + class PolicyService: """Policy operations with service-owned ORM transaction boundaries.""" @@ -63,6 +84,25 @@ def get_policy_json( policy = self.get_policy(country_id, policy_id) return None if policy is None else policy.policy_json + def get_policy_snapshot( + self, + country_id: str, + policy_id: int, + ) -> LegacyPolicySnapshot | None: + """Return detached fields required to mirror one existing v1 policy.""" + + policy = self.get_policy(country_id, policy_id) + if policy is None: + return None + return LegacyPolicySnapshot( + country_id=policy.country_id, + legacy_policy_id=policy.id, + label=policy.label, + api_version=policy.api_version, + policy_json=copy.deepcopy(policy.policy_json), + source_policy_hash=policy.policy_hash, + ) + def search_policies( self, country_id: str, @@ -96,20 +136,34 @@ def set_policy( country_id: str, label: str | None, policy_json: dict, - ) -> tuple[int, str, bool]: + ) -> PolicySetResult: country_id = country_id.lower() if country_id not in COUNTRY_PACKAGE_VERSIONS: raise ValueError(f"Invalid country_id: {country_id}") policy_hash = hash_object(policy_json) with self._sessions.begin() as session: - return self._set_policy( + policy, message, is_existing_policy = self._set_policy( session, country_id, label, policy_json, policy_hash, ) + snapshot = LegacyPolicySnapshot( + country_id=policy.country_id, + legacy_policy_id=policy.id, + label=policy.label, + api_version=policy.api_version, + policy_json=copy.deepcopy(policy.policy_json), + source_policy_hash=policy.policy_hash, + ) + return PolicySetResult( + policy_id=policy.id, + message=message, + is_existing_policy=is_existing_policy, + snapshot=snapshot, + ) def _set_policy( self, @@ -118,7 +172,7 @@ def _set_policy( label: str | None, policy_json: dict, policy_hash: str, - ) -> tuple[int, str, bool]: + ) -> tuple[Policy, str, bool]: existing = self._get_unique_policy_with_label( session, country_id, @@ -126,7 +180,7 @@ def _set_policy( label or None, ) if existing is not None: - return existing.id, "Policy already exists", True + return existing, "Policy already exists", True policy = Policy( country_id=country_id, @@ -137,7 +191,7 @@ def _set_policy( ) session.add(policy) session.flush() - return policy.id, "Policy created", False + return policy, "Policy created", False def _create_new_policy( self, diff --git a/policyengine_api/services/user_policy_mirroring.py b/policyengine_api/services/user_policy_mirroring.py new file mode 100644 index 000000000..5da708794 --- /dev/null +++ b/policyengine_api/services/user_policy_mirroring.py @@ -0,0 +1,301 @@ +"""Immediate mirroring of committed v1 saved policies into v2 associations.""" + +from __future__ import annotations + +from collections.abc import Callable +import time +from typing import Protocol + +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.data.v2.policies.catalog import PolicyCatalogValidationError +from policyengine_api.data.v2.policies.legacy import ( + LegacyPolicyMappingIntegrityError, + LegacyPolicySnapshot, + LegacyPolicyTranslationError, +) +from policyengine_api.data.v2.policies.persistence import ( + PolicyContentHashCollisionError, + PolicyPersistenceIntegrityError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.data.v2.user_policies.legacy import ( + LegacyUserPolicyIntegrityError, + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, +) +from policyengine_api.gcp_logging import logger +from policyengine_api.services.policy_service import PolicyService +from policyengine_api.services.user_policy_service import ( + PendingUserPolicyMirrorEvent, + UserPolicyService, +) + + +class LegacyUserPolicyMirror(Protocol): + def mirror_legacy_user_policy( + self, + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str], + ) -> LegacyUserPolicyPersistenceResult: ... + + +class UserPolicyMirrorUnavailableError(RuntimeError): + """Raised when a committed saved policy could not be mirrored immediately.""" + + +def _default_mirror_factory() -> LegacyUserPolicyMirror: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.data.v2.user_policies.service import V2UserPolicyService + + return V2UserPolicyService(get_v2_session_factory()) + + +def _failure_category(error: Exception) -> str: + if isinstance(error, V2ConfigurationError): + return "configuration" + if isinstance( + error, + ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, + PolicyCatalogValidationError, + LegacyPolicyTranslationError, + ), + ): + return "catalog_or_translation" + if isinstance( + error, + ( + LegacyPolicyMappingIntegrityError, + LegacyUserPolicyIntegrityError, + PolicyContentHashCollisionError, + PolicyPersistenceIntegrityError, + ), + ): + return "integrity" + if isinstance(error, SQLAlchemyError): + return "database" + return "unexpected" + + +def _log_mirror_operation( + *, + country_id: str, + legacy_user_policy_id: int, + source_revision: int | None, + requested_through_revision: int, + started_at: float, + outcome: str, + actual_write_sources: list[str], + result: LegacyUserPolicyPersistenceResult | None = None, + failure_category: str | None = None, +) -> None: + logger.log_struct( + { + "message": "V1 saved-policy immediate mirror completed", + "metric_name": "v1_user_policy_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": actual_write_sources, + "country_id": country_id, + "legacy_user_policy_id": legacy_user_policy_id, + "source_revision": source_revision, + "requested_through_revision": requested_through_revision, + "destination_association_id": ( + str(result.association_id) if result is not None else None + ), + "destination_policy_id": ( + str(result.policy_id) if result is not None else None + ), + "outcome": outcome, + "failure_category": failure_category, + "association_created": ( + result.association_created if result is not None else None + ), + "association_updated": ( + result.association_updated if result is not None else None + ), + "mapping_created": result.mapping_created if result is not None else None, + "duration_ms": round((time.perf_counter() - started_at) * 1000, 3), + }, + severity="INFO" if outcome == "ok" else "ERROR", + ) + + +def _run_user_policy_mirror( + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str], + mirror_factory: Callable[[], LegacyUserPolicyMirror] | None, +) -> LegacyUserPolicyPersistenceResult: + return (mirror_factory or _default_mirror_factory)().mirror_legacy_user_policy( + snapshot, + reform_snapshot, + source_revision=source_revision, + changed_fields=changed_fields, + ) + + +def mirror_user_policy_after_commit( + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str] = frozenset(), + mirror_factory: Callable[[], LegacyUserPolicyMirror] | None = None, +) -> LegacyUserPolicyPersistenceResult: + """Require one complete Supabase association transaction before v1 success.""" + + started_at = time.perf_counter() + try: + result = _run_user_policy_mirror( + snapshot, + reform_snapshot, + source_revision=source_revision, + changed_fields=changed_fields, + mirror_factory=mirror_factory, + ) + except Exception as error: + _log_mirror_operation( + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + source_revision=source_revision, + requested_through_revision=source_revision, + started_at=started_at, + outcome="error", + actual_write_sources=["cloud_sql"], + failure_category=_failure_category(error), + ) + raise UserPolicyMirrorUnavailableError( + "The committed v1 saved policy could not be mirrored to v2" + ) from error + + _log_mirror_operation( + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + source_revision=source_revision, + requested_through_revision=source_revision, + started_at=started_at, + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + result=result, + ) + return result + + +def mirror_pending_user_policy_events_after_commit( + country_id: str, + legacy_user_policy_id: int, + *, + through_revision: int, + event_service: UserPolicyService | None = None, + reform_snapshot_loader: Callable[[str, int], LegacyPolicySnapshot | None] + | None = None, + mirror_factory: Callable[[], LegacyUserPolicyMirror] | None = None, +) -> LegacyUserPolicyPersistenceResult: + """Synchronously apply retained source events through one request's revision.""" + + selected_event_service = event_service or UserPolicyService() + selected_reform_loader = ( + reform_snapshot_loader or PolicyService().get_policy_snapshot + ) + request_started_at = time.perf_counter() + active_event: PendingUserPolicyMirrorEvent | None = None + event_started_at: dict[int, float] = {} + destination_results: dict[int, LegacyUserPolicyPersistenceResult] = {} + + def process( + event: PendingUserPolicyMirrorEvent, + ) -> LegacyUserPolicyPersistenceResult: + nonlocal active_event + + active_event = event + event_started_at[event.event_id] = time.perf_counter() + reform_snapshot = selected_reform_loader( + event.snapshot.country_id, + event.snapshot.reform_id, + ) + if reform_snapshot is None: + raise UserPolicyMirrorUnavailableError( + "The saved policy references an unavailable reform policy" + ) + result = _run_user_policy_mirror( + event.snapshot, + reform_snapshot, + source_revision=event.source_revision, + changed_fields=event.changed_fields, + mirror_factory=mirror_factory, + ) + destination_results[event.event_id] = result + return result + + def after_processed_commit( + event: PendingUserPolicyMirrorEvent, + result: LegacyUserPolicyPersistenceResult, + ) -> None: + nonlocal active_event + + _log_mirror_operation( + country_id=event.snapshot.country_id, + legacy_user_policy_id=event.snapshot.legacy_user_policy_id, + source_revision=event.source_revision, + requested_through_revision=through_revision, + started_at=event_started_at.pop(event.event_id), + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + result=result, + ) + destination_results.pop(event.event_id, None) + active_event = None + + try: + return selected_event_service.process_pending_mirror_events( + country_id, + legacy_user_policy_id, + through_revision=through_revision, + processor=process, + after_processed_commit=after_processed_commit, + ) + except Exception as error: + event = active_event + result = destination_results.get(event.event_id) if event is not None else None + failure = error + if isinstance(error, UserPolicyMirrorUnavailableError) and isinstance( + error.__cause__, Exception + ): + failure = error.__cause__ + _log_mirror_operation( + country_id=(event.snapshot.country_id if event is not None else country_id), + legacy_user_policy_id=( + event.snapshot.legacy_user_policy_id + if event is not None + else legacy_user_policy_id + ), + source_revision=event.source_revision if event is not None else None, + requested_through_revision=through_revision, + started_at=( + event_started_at.get(event.event_id, request_started_at) + if event is not None + else request_started_at + ), + outcome="error", + actual_write_sources=( + ["cloud_sql", "supabase"] if result is not None else ["cloud_sql"] + ), + result=result, + failure_category=_failure_category(failure), + ) + raise UserPolicyMirrorUnavailableError( + "The committed v1 saved policy event could not be mirrored to v2" + ) from error diff --git a/policyengine_api/services/user_policy_service.py b/policyengine_api/services/user_policy_service.py index 07339310f..a67b8d54a 100644 --- a/policyengine_api/services/user_policy_service.py +++ b/policyengine_api/services/user_policy_service.py @@ -1,13 +1,27 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass +from datetime import datetime, timezone +from enum import StrEnum from typing import Any, Mapping from sqlalchemy import select +from sqlalchemy.exc import ( + IntegrityError, + OperationalError, + SQLAlchemyError, + TimeoutError as SQLAlchemyTimeoutError, +) from sqlalchemy.orm import Session, sessionmaker from policyengine_api.data.orm import get_v1_session_factory -from policyengine_api.data.v1_models import UserPolicy +from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent +from policyengine_api.data.v2.user_policies.legacy import ( + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, + fingerprint_legacy_user_policy, +) USER_POLICY_IDENTITY_FIELDS = ( @@ -21,12 +35,87 @@ "baseline_label", "dataset", ) +USER_POLICY_MIRROR_PAYLOAD_SCHEMA_VERSION = 1 +USER_POLICY_MUTABLE_FIELDS = frozenset( + { + "reform_label", + "baseline_label", + "year", + "geography", + "dataset", + "number_of_provisions", + "api_version", + "added_date", + "updated_date", + "budgetary_impact", + "type", + } +) + + +class UserPolicyMirrorEventIntegrityError(RuntimeError): + """Raised when durable saved-policy mirror input is invalid or inconsistent.""" + + +class UserPolicyPersistenceFailureCategory(StrEnum): + TIMEOUT = "timeout" + UNAVAILABLE = "unavailable" + INTEGRITY = "integrity" + DATABASE = "database" + UNEXPECTED = "unexpected" + + +class UserPolicyPersistenceError(RuntimeError): + """Domain error containing only safe persistence-failure attributes.""" + + def __init__(self, category: UserPolicyPersistenceFailureCategory) -> None: + super().__init__("Saved-policy persistence failed") + self.category = UserPolicyPersistenceFailureCategory(category) + + @property + def retryable(self) -> bool: + return self.category in { + UserPolicyPersistenceFailureCategory.TIMEOUT, + UserPolicyPersistenceFailureCategory.UNAVAILABLE, + } + + @classmethod + def from_exception(cls, error: Exception) -> UserPolicyPersistenceError: + if isinstance(error, SQLAlchemyTimeoutError): + return cls(UserPolicyPersistenceFailureCategory.TIMEOUT) + if isinstance(error, OperationalError): + return cls(UserPolicyPersistenceFailureCategory.UNAVAILABLE) + if isinstance(error, IntegrityError): + return cls(UserPolicyPersistenceFailureCategory.INTEGRITY) + if isinstance(error, SQLAlchemyError): + return cls(UserPolicyPersistenceFailureCategory.DATABASE) + return cls(UserPolicyPersistenceFailureCategory.UNEXPECTED) + + +@dataclass(frozen=True) +class PendingUserPolicyMirrorEvent: + event_id: int + source_revision: int + event_type: str + snapshot: LegacyUserPolicySnapshot + changed_fields: frozenset[str] + source_fingerprint_sha256: str @dataclass(frozen=True) class UserPolicyCreateResult: user_policy: UserPolicy created: bool + snapshot: LegacyUserPolicySnapshot + mirror_revision: int | None = None + + +@dataclass(frozen=True) +class UserPolicyUpdateResult: + user_policy: UserPolicy + snapshot: LegacyUserPolicySnapshot + changed_fields: frozenset[str] + mirror_revision: int | None = None class UserPolicyService: @@ -42,35 +131,177 @@ def __init__( def _sessions(self) -> sessionmaker[Session]: return self._injected_session_factory or get_v1_session_factory() + @staticmethod + def _snapshot(user_policy: UserPolicy) -> LegacyUserPolicySnapshot: + return LegacyUserPolicySnapshot( + country_id=user_policy.country_id, + legacy_user_policy_id=user_policy.id, + reform_id=user_policy.reform_id, + reform_label=user_policy.reform_label, + baseline_id=user_policy.baseline_id, + baseline_label=user_policy.baseline_label, + user_id=user_policy.user_id, + year=user_policy.year, + geography=user_policy.geography, + dataset=user_policy.dataset, + number_of_provisions=user_policy.number_of_provisions, + api_version=user_policy.api_version, + added_date=user_policy.added_date, + updated_date=user_policy.updated_date, + budgetary_impact=user_policy.budgetary_impact, + type=user_policy.type, + ) + @staticmethod def _find_matching_user_policy( session: Session, values: Mapping[str, Any], + *, + lock: bool, ) -> UserPolicy | None: - return session.scalar( - select(UserPolicy).where( - *( - getattr(UserPolicy, field) == values[field] - for field in USER_POLICY_IDENTITY_FIELDS - ) + statement = select(UserPolicy).where( + *( + getattr(UserPolicy, field) == values[field] + for field in USER_POLICY_IDENTITY_FIELDS ) ) + if lock: + statement = statement.with_for_update() + return session.scalar(statement) + + @classmethod + def _record_mirror_event( + cls, + session: Session, + user_policy: UserPolicy, + *, + event_type: str, + changed_fields: frozenset[str], + ) -> int: + user_policy.mirror_revision += 1 + snapshot = cls._snapshot(user_policy) + event = UserPolicyMirrorEvent( + country_id=user_policy.country_id, + legacy_user_policy_id=user_policy.id, + source_revision=user_policy.mirror_revision, + event_type=event_type, + payload_schema_version=USER_POLICY_MIRROR_PAYLOAD_SCHEMA_VERSION, + payload_json={ + "snapshot": snapshot.model_dump(mode="json"), + "changed_fields": sorted(changed_fields), + }, + source_fingerprint_sha256=fingerprint_legacy_user_policy(snapshot), + ) + session.add_all((user_policy, event)) + session.flush() + return user_policy.mirror_revision + + @staticmethod + def _decode_mirror_event( + event: UserPolicyMirrorEvent, + ) -> PendingUserPolicyMirrorEvent: + if event.payload_schema_version != USER_POLICY_MIRROR_PAYLOAD_SCHEMA_VERSION: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event payload version is unsupported" + ) + payload = event.payload_json + if not isinstance(payload, dict) or set(payload) != { + "snapshot", + "changed_fields", + }: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event payload has an invalid shape" + ) + changed_fields = payload["changed_fields"] + if not isinstance(changed_fields, list) or not all( + isinstance(field, str) for field in changed_fields + ): + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event changed fields are invalid" + ) + if ( + changed_fields != sorted(set(changed_fields)) + or not set(changed_fields) <= USER_POLICY_MUTABLE_FIELDS + ): + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event changed fields are unsupported" + ) + try: + snapshot = LegacyUserPolicySnapshot.model_validate(payload["snapshot"]) + except (TypeError, ValueError) as error: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event snapshot is invalid" + ) from error + if ( + snapshot.country_id != event.country_id + or snapshot.legacy_user_policy_id != event.legacy_user_policy_id + ): + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event source identity conflicts with its payload" + ) + fingerprint = fingerprint_legacy_user_policy(snapshot) + if fingerprint != event.source_fingerprint_sha256: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event fingerprint conflicts with its payload" + ) + if event.event_type not in {"create", "update"}: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event type is unsupported" + ) + if event.event_type == "create" and changed_fields: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy create mirror event contains changed fields" + ) + if event.event_type == "update" and not changed_fields: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy update mirror event has no changed fields" + ) + return PendingUserPolicyMirrorEvent( + event_id=event.id, + source_revision=event.source_revision, + event_type=event.event_type, + snapshot=snapshot, + changed_fields=frozenset(changed_fields), + source_fingerprint_sha256=fingerprint, + ) def create_or_get_user_policy( self, values: Mapping[str, Any], + *, + record_mirror_event: bool = False, ) -> UserPolicyCreateResult: - with self._sessions.begin() as session: - user_policy = self._find_matching_user_policy(session, values) - created = user_policy is None - if user_policy is None: - user_policy = UserPolicy(**values) - session.add(user_policy) - session.flush() - return UserPolicyCreateResult( - user_policy=user_policy, - created=created, - ) + try: + with self._sessions.begin() as session: + user_policy = self._find_matching_user_policy( + session, + values, + lock=record_mirror_event, + ) + created = user_policy is None + if user_policy is None: + user_policy = UserPolicy(**values) + session.add(user_policy) + session.flush() + mirror_revision = None + if record_mirror_event: + mirror_revision = self._record_mirror_event( + session, + user_policy, + event_type="create", + changed_fields=frozenset(), + ) + result = UserPolicyCreateResult( + user_policy=user_policy, + created=created, + snapshot=self._snapshot(user_policy), + mirror_revision=mirror_revision, + ) + except UserPolicyPersistenceError: + raise + except Exception as error: + raise UserPolicyPersistenceError.from_exception(error) from error + return result def list_user_policies( self, @@ -92,16 +323,113 @@ def update_user_policy( country_id: str, user_policy_id: int, values: Mapping[str, Any], - ) -> UserPolicy | None: - with self._sessions.begin() as session: - user_policy = session.scalar( - select(UserPolicy).where( + *, + record_mirror_event: bool = False, + ) -> UserPolicyUpdateResult | None: + try: + with self._sessions.begin() as session: + statement = select(UserPolicy).where( UserPolicy.id == user_policy_id, UserPolicy.country_id == country_id, ) - ) - if user_policy is None: - return None - for field, value in values.items(): - setattr(user_policy, field, value) - return user_policy + if record_mirror_event: + statement = statement.with_for_update() + user_policy = session.scalar(statement) + if user_policy is None: + return None + for field, value in values.items(): + setattr(user_policy, field, value) + session.flush() + changed_fields = frozenset(values) + mirror_revision = None + if record_mirror_event: + mirror_revision = self._record_mirror_event( + session, + user_policy, + event_type="update", + changed_fields=changed_fields, + ) + result = UserPolicyUpdateResult( + user_policy=user_policy, + snapshot=self._snapshot(user_policy), + changed_fields=changed_fields, + mirror_revision=mirror_revision, + ) + except UserPolicyPersistenceError: + raise + except Exception as error: + raise UserPolicyPersistenceError.from_exception(error) from error + return result + + def process_pending_mirror_events( + self, + country_id: str, + legacy_user_policy_id: int, + *, + through_revision: int, + processor: Callable[ + [PendingUserPolicyMirrorEvent], + LegacyUserPolicyPersistenceResult, + ], + after_processed_commit: Callable[ + [ + PendingUserPolicyMirrorEvent, + LegacyUserPolicyPersistenceResult, + ], + None, + ] + | None = None, + ) -> LegacyUserPolicyPersistenceResult: + """Process retained source events in order through one request's revision.""" + + latest_result: LegacyUserPolicyPersistenceResult | None = None + while True: + with self._sessions.begin() as session: + event = session.scalar( + select(UserPolicyMirrorEvent) + .where( + UserPolicyMirrorEvent.country_id == country_id, + UserPolicyMirrorEvent.legacy_user_policy_id + == legacy_user_policy_id, + UserPolicyMirrorEvent.source_revision <= through_revision, + UserPolicyMirrorEvent.processed_at.is_(None), + ) + .order_by(UserPolicyMirrorEvent.source_revision) + .limit(1) + .with_for_update() + ) + if event is None: + break + pending = self._decode_mirror_event(event) + processed_result = processor(pending) + event.processed_at = datetime.now(timezone.utc).replace(tzinfo=None) + session.add(event) + session.flush() + latest_result = processed_result + if after_processed_commit is not None: + after_processed_commit(pending, processed_result) + if pending.source_revision == through_revision: + break + if latest_result is None: + with self._sessions.begin() as session: + event = session.scalar( + select(UserPolicyMirrorEvent) + .where( + UserPolicyMirrorEvent.country_id == country_id, + UserPolicyMirrorEvent.legacy_user_policy_id + == legacy_user_policy_id, + UserPolicyMirrorEvent.source_revision == through_revision, + UserPolicyMirrorEvent.processed_at.is_not(None), + ) + .with_for_update() + ) + if event is None: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror request has no retained event" + ) + pending = self._decode_mirror_event(event) + processed_result = processor(pending) + latest_result = processed_result + if after_processed_commit is not None: + after_processed_commit(pending, processed_result) + return latest_result diff --git a/scripts/guards/migration_contracts.py b/scripts/guards/migration_contracts.py index 699991520..04d3c0918 100644 --- a/scripts/guards/migration_contracts.py +++ b/scripts/guards/migration_contracts.py @@ -83,11 +83,14 @@ def _check_workflows(payload: dict[str, Any]) -> list[str]: violations.append( f"{context}: unknown route_group {request['route_group']!r}" ) - if not request["stable_response_fields"]: + if ( + not request["stable_response_fields"] + and request["expected_status"] != 204 + ): violations.append(f"{context}: stable_response_fields is required") if not request["path"].startswith("/"): violations.append(f"{context}: path must start with /") - if request["expected_status"] not in {200, 201, 202}: + if request["expected_status"] not in {200, 201, 202, 204}: violations.append( f"{context}: unexpected status {request['expected_status']}" ) diff --git a/scripts/qualify_v2_policy_migration.py b/scripts/qualify_v2_policy_migration.py new file mode 100644 index 000000000..32fc3ea5c --- /dev/null +++ b/scripts/qualify_v2_policy_migration.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Verify that a Supabase target has no retained v2 policy data.""" + +from policyengine_api.data.v2.policy_migration_qualification import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/contract/registry.py b/tests/contract/registry.py index c24e9eb41..92dcb494d 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -49,6 +49,177 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="policy_resources_v2", + current_contract="typed_v2_resources", + future_owner_pr="PR 10: Policy Migration", + requests=( + ContractRequest( + method="POST", + path="/v2/policies?country_id=us", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/v2/policies/{policy_id}?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/v2/policies?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.items", + "result.offset", + "result.limit", + "result.has_more", + ), + route_group="policy", + ), + ), + ), + WorkflowContract( + name="saved_policy_v1_compatibility", + current_contract="api_v1_compatible", + future_owner_pr="PR 10: Policy Migration", + requests=( + ContractRequest( + method="POST", + path="/us/user-policy", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.id", + "result.reform_id", + "result.reform_label", + "result.baseline_id", + "result.user_id", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/us/user-policy/{user_id}", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result", + ), + route_group="policy", + ), + ContractRequest( + method="PUT", + path="/us/user-policy", + expected_status=200, + stable_response_fields=("status", "message", "result.id"), + route_group="policy", + ), + ), + ), + WorkflowContract( + name="user_policy_associations_v2", + current_contract="typed_v2_resources", + future_owner_pr="PR 10: Policy Migration", + requests=( + ContractRequest( + method="POST", + path="/v2/user-policies?country_id=us", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.user_id", + "result.item.policy_id", + "result.item.name", + "result.item.description", + "result.item.created_at", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path=("/v2/user-policies/{association_id}?country_id=us"), + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.user_id", + "result.item.policy_id", + "result.item.name", + "result.item.description", + "result.item.created_at", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/v2/user-policies?country_id=us&user_id=caller", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.items", + "result.offset", + "result.limit", + "result.has_more", + ), + route_group="policy", + ), + ContractRequest( + method="PATCH", + path=("/v2/user-policies/{association_id}?country_id=us"), + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.name", + "result.item.description", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="DELETE", + path=("/v2/user-policies/{association_id}?country_id=us"), + expected_status=204, + stable_response_fields=(), + route_group="policy", + ), + ), + ), WorkflowContract( name="household_save_edit_read", current_contract="api_v1_compatible", diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py index 8a1991d84..4545af6bf 100644 --- a/tests/contract/test_app_v2_workflow_contracts.py +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -9,6 +9,9 @@ def test_app_v2_workflow_contract_registry_is_complete(): assert {workflow.name for workflow in APP_V2_WORKFLOW_CONTRACTS} == { "policy_save_search", + "policy_resources_v2", + "saved_policy_v1_compatibility", + "user_policy_associations_v2", "household_save_edit_read", "household_calculate", "region_selection", @@ -21,7 +24,12 @@ def test_app_v2_workflow_contract_registry_is_complete(): for workflow in APP_V2_WORKFLOW_CONTRACTS: expected_contract = ( "typed_v2_resources" - if workflow.name == "metadata_resources_v2_preview" + if workflow.name + in { + "metadata_resources_v2_preview", + "policy_resources_v2", + "user_policy_associations_v2", + } else "api_v1_compatible" ) assert workflow.current_contract == expected_contract @@ -29,10 +37,10 @@ def test_app_v2_workflow_contract_registry_is_complete(): assert workflow.requests for request in APP_V2_ROUTE_CONTRACTS: - assert request.method in {"GET", "POST", "PUT", "PATCH"} + assert request.method in {"GET", "POST", "PUT", "PATCH", "DELETE"} assert request.path.startswith("/") - assert request.expected_status in {200, 201, 202} - assert request.stable_response_fields + assert request.expected_status in {200, 201, 202, 204} + assert request.stable_response_fields or request.expected_status == 204 assert request.route_group in ROUTE_GROUP_CONFIG_BY_NAME assert all( @@ -44,6 +52,6 @@ def test_app_v2_workflow_contract_registry_is_complete(): } == { request.path for workflow in APP_V2_WORKFLOW_CONTRACTS - if workflow.name == "metadata_resources_v2_preview" + if workflow.current_contract == "typed_v2_resources" for request in workflow.requests } diff --git a/tests/contract/test_policy_v2_compatibility.py b/tests/contract/test_policy_v2_compatibility.py new file mode 100644 index 000000000..8962623c8 --- /dev/null +++ b/tests/contract/test_policy_v2_compatibility.py @@ -0,0 +1,58 @@ +"""Compatibility boundaries between app-v2 labels and native v2 resources.""" + +from __future__ import annotations + +from uuid import UUID + +import pytest + +from policyengine_api.data.v2.models import Policy +from policyengine_api.data.v2.policies.api_schemas import PolicyCreateRequest +from policyengine_api.data.v2.user_policies.legacy import ( + LegacyUserPolicySnapshot, + project_legacy_user_policy, +) + + +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") + + +def test_app_v2_reform_label_maps_to_association_name_only() -> None: + snapshot = LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=10, + reform_id=2, + reform_label="User-visible app label", + baseline_id=1, + baseline_label="Current law", + user_id="auth0|one", + year="2026", + geography="us", + dataset=None, + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + + projection = project_legacy_user_policy(snapshot, policy_id=POLICY_ID) + + assert projection.name == "User-visible app label" + assert projection.description is None + assert projection.policy_id == POLICY_ID + assert "name" not in Policy.__table__.c + assert "description" not in Policy.__table__.c + + +def test_core_policy_request_rejects_association_presentation_fields() -> None: + with pytest.raises(ValueError): + PolicyCreateRequest.model_validate( + { + "country_id": "us", + "tax_benefit_model_id": str(POLICY_ID), + "parameter_values": [], + "name": "User-visible app label", + } + ) diff --git a/tests/contract/test_v1_route_contracts.py b/tests/contract/test_v1_route_contracts.py index bc06f381f..66915b3e3 100644 --- a/tests/contract/test_v1_route_contracts.py +++ b/tests/contract/test_v1_route_contracts.py @@ -14,7 +14,9 @@ ReportOutput, ReportOutputRun, Simulation, + UserPolicy, ) +from policyengine_api.data.v2.user_policies.legacy import LegacyUserPolicySnapshot from policyengine_api.extensions import cache from policyengine_api.routes.household_routes import household_bp from policyengine_api.routes.policy_routes import policy_bp @@ -28,6 +30,10 @@ HouseholdCalculationResult, ) from policyengine_api.services.simulation_service import SimulationCreateResult +from policyengine_api.services.user_policy_service import ( + UserPolicyCreateResult, + UserPolicyUpdateResult, +) from tests.contract.clients import ( ASGIContractClient, ContractClient, @@ -184,12 +190,30 @@ def _resolved_path(path: str) -> str: .replace("{household_id}", "456") .replace("{simulation_id}", "11") .replace("{report_id}", "33") + .replace("{user_id}", "auth0|one") ) def _json_payload(contract: ContractRequest) -> dict | None: if contract.path == "/us/policy": return {"label": "Utah reform", "data": {"gov.example.parameter": 1}} + if contract.path == "/us/user-policy" and contract.method == "POST": + return { + "reform_id": 22, + "reform_label": "Tax reform", + "baseline_id": 2, + "baseline_label": "Current law", + "user_id": "auth0|one", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1", + "added_date": 1, + "updated_date": 2, + } + if contract.path == "/us/user-policy" and contract.method == "PUT": + return {"id": 10, "reform_label": "Updated tax reform"} if contract.path == "/us/household": return {"label": "Empty household", "data": {}} if contract.path == "/us/household/{household_id}": @@ -219,6 +243,42 @@ def _fake_country(): def _patched_route_dependencies(): stack = ExitStack() + saved_policy = UserPolicy( + id=10, + country_id="us", + reform_id=22, + reform_label="Tax reform", + baseline_id=2, + baseline_label="Current law", + user_id="auth0|one", + year="2026", + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + saved_snapshot = LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=10, + reform_id=22, + reform_label="Tax reform", + baseline_id=2, + baseline_label="Current law", + user_id="auth0|one", + year="2026", + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) stack.enter_context( patch( "policyengine_api.routes.policy_routes.policy_service.get_policy", @@ -232,6 +292,32 @@ def _patched_route_dependencies(): ), ) ) + stack.enter_context( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.create_or_get_user_policy", + return_value=UserPolicyCreateResult( + user_policy=saved_policy, + created=True, + snapshot=saved_snapshot, + ), + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.list_user_policies", + return_value=[saved_policy], + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.update_user_policy", + return_value=UserPolicyUpdateResult( + user_policy=saved_policy, + snapshot=saved_snapshot, + changed_fields=frozenset({"reform_label"}), + ), + ) + ) stack.enter_context( patch( "policyengine_api.routes.policy_routes.policy_service.set_policy", @@ -397,6 +483,30 @@ def _expected_subset(contract: ContractRequest) -> dict: "message": "Policies found", "result": [{"id": 123, "label": "Tax reform"}], } + if contract.path == "/us/user-policy" and contract.method == "POST": + return { + "status": "ok", + "message": "Record created successfully", + "result": { + "id": 10, + "reform_id": 22, + "reform_label": "Tax reform", + "baseline_id": 2, + "user_id": "auth0|one", + }, + } + if contract.path == "/us/user-policy/{user_id}": + return { + "status": "ok", + "message": None, + "result": [{"id": 10, "reform_label": "Tax reform"}], + } + if contract.path == "/us/user-policy" and contract.method == "PUT": + return { + "status": "ok", + "message": "Record updated successfully", + "result": {"id": 10}, + } if contract.path == "/us/household": return {"status": "ok", "message": None, "result": {"household_id": 456}} if contract.path == "/us/household/{household_id}" and contract.method == "PUT": diff --git a/tests/integration/test_alembic_mysql_lifecycle.py b/tests/integration/test_alembic_mysql_lifecycle.py index 9e4f0344f..4428d45f3 100644 --- a/tests/integration/test_alembic_mysql_lifecycle.py +++ b/tests/integration/test_alembic_mysql_lifecycle.py @@ -30,7 +30,7 @@ BASELINE_REVISION = "eafc2a547a4e" -PREVIOUS_REVISION = "17bb32415f97" +PREVIOUS_REVISION = "1914c0422236" def _deployed_question_table() -> Table: @@ -97,6 +97,10 @@ def test_fresh_upgrade_check_downgrade_and_reupgrade(): } assert reform_impact_columns["dataset"]["default"] is None assert reform_impact_columns["execution_id"]["nullable"] is False + assert "mirror_revision" in { + column["name"] for column in inspector.get_columns("user_policies") + } + assert "user_policy_mirror_events" in inspector.get_table_names() command.downgrade(config, BASELINE_REVISION) assert "question" in inspect(engine).get_table_names() @@ -142,6 +146,10 @@ def test_pending_upgrade_commits_head_revision(monkeypatch): } assert reform_impact_columns["dataset"]["default"] is None assert reform_impact_columns["execution_id"]["nullable"] is False + assert "mirror_revision" in { + column["name"] for column in inspector.get_columns("user_policies") + } + assert "user_policy_mirror_events" in inspector.get_table_names() finally: command.upgrade(config, "head") engine.dispose() diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index dd24c82fd..fe5358147 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -23,7 +23,8 @@ BASELINE_REVISION = "f5ef4347cb2a" -HEAD_REVISION = "68b4a5ae5dc5" +PREVIOUS_HEAD_REVISION = "711ec2f0a5a5" +HEAD_REVISION = "c21c4a807a49" V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) @@ -76,6 +77,20 @@ def _assert_head(engine) -> None: ] assert canonical_index["unique"] assert canonical_index["column_names"] == ["parameter_id", "start_date"] + policy_value_constraint = next( + constraint + for constraint in inspect(engine).get_unique_constraints("parameter_values") + if constraint["name"] == "uq_parameter_values_policy_parameter_start_date" + ) + assert policy_value_constraint["column_names"] == [ + "policy_id", + "parameter_id", + "start_date", + ] + assert { + "legacy_policy_mappings", + "legacy_user_policy_mappings", + } <= set(inspect(engine).get_table_names(schema="public")) def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: @@ -93,6 +108,22 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: command.check(config) _assert_head(engine) + command.downgrade(config, PREVIOUS_HEAD_REVISION) + with engine.connect() as connection: + context = MigrationContext.configure(connection) + assert context.get_current_revision() == PREVIOUS_HEAD_REVISION + assert "last_applied_source_revision" not in { + column["name"] + for column in inspect(engine).get_columns( + "legacy_user_policy_mappings", + schema="public", + ) + } + + command.upgrade(config, "head") + command.check(config) + _assert_head(engine) + command.downgrade(config, BASELINE_REVISION) with engine.connect() as connection: context = MigrationContext.configure(connection) diff --git a/tests/integration/test_v1_policy_dual_write.py b/tests/integration/test_v1_policy_dual_write.py new file mode 100644 index 000000000..6d16a5ba6 --- /dev/null +++ b/tests/integration/test_v1_policy_dual_write.py @@ -0,0 +1,259 @@ +"""Cross-database transaction tests for immediate v1 policy mirroring.""" + +from __future__ import annotations + +import os + +import pytest +from sqlalchemy import create_engine, delete, func, select +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v1_models import Policy as V1Policy +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.data.v2.policies.legacy import persist_legacy_policy +from policyengine_api.data.v2.policies.service import V2PolicyService +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.services.policy_mirroring import ( + PolicyMirrorUnavailableError, + mirror_policy_after_commit, +) +from policyengine_api.services.policy_service import PolicyService + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get( + V2_ALEMBIC_DISPOSABLE_TEST, + "", + ), + } + ) + if not settings.disposable_test: + pytest.fail("dual-write tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +def _v1_service(): + engine = create_engine("sqlite://") + with engine.begin() as connection: + connection.exec_driver_sql( + """ + CREATE TABLE policy ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + country_id VARCHAR(3) NOT NULL, + label VARCHAR(255), + api_version VARCHAR(10) NOT NULL, + policy_json JSON NOT NULL, + policy_hash VARCHAR(255) NOT NULL + ) + """ + ) + sessions = sessionmaker(engine, expire_on_commit=False) + return engine, PolicyService(sessions) + + +def _seed_catalog(v2_sessions) -> tuple[object, str]: + with v2_sessions.begin() as session: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version=POLICYENGINE_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.phase10.cross_database_rate", + tax_benefit_model_version=version, + ) + session.add(parameter) + session.flush() + return model.id, parameter.name + + +def _cleanup_v2(v2_engine, model_id) -> None: + if model_id is None: + return + with v2_engine.begin() as connection: + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + connection.execute( + delete(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id.in_(policy_ids) + ) + ) + connection.execute( + delete(ParameterValue).where(ParameterValue.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def _create_v1(service: PolicyService, parameter_name: str): + return service.set_policy( + "us", + "Cross-database policy", + {parameter_name: {"2026": 0.2}}, + ) + + +def test_both_commits_and_interrupted_response_retry_resolve_one_mapping() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, v1_service = _v1_service() + model_id = None + try: + model_id, parameter_name = _seed_catalog(v2_sessions) + mirror_service = V2PolicyService(v2_sessions) + creation = _create_v1(v1_service, parameter_name) + first = mirror_policy_after_commit( + creation.snapshot, + mirror_factory=lambda: mirror_service, + ) + + # Simulate losing the HTTP response after both commits by repeating the + # exact create and mirror operations. + retry_creation = _create_v1(v1_service, parameter_name) + retry = mirror_policy_after_commit( + retry_creation.snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert creation.is_existing_policy is False + assert retry_creation.is_existing_policy is True + assert retry.policy_id == first.policy_id + with v2_sessions() as session: + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 1 + ) + assert session.scalar(select(func.count()).select_from(Policy)) == 1 + assert v1_service.get_policy("us", creation.policy_id) is not None + finally: + _cleanup_v2(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() + + +def test_catalog_failure_leaves_cloud_sql_committed_and_retry_completes() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, v1_service = _v1_service() + model_id = None + parameter_name = "gov.phase10.cross_database_rate" + try: + creation = _create_v1(v1_service, parameter_name) + mirror_service = V2PolicyService(v2_sessions) + + with pytest.raises(PolicyMirrorUnavailableError): + mirror_policy_after_commit( + creation.snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert v1_service.get_policy("us", creation.policy_id) is not None + with v2_sessions() as session: + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 0 + ) + + model_id, _parameter_name = _seed_catalog(v2_sessions) + retry_creation = _create_v1(v1_service, parameter_name) + result = mirror_policy_after_commit( + retry_creation.snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert retry_creation.is_existing_policy is True + assert result.mapping_created is True + finally: + _cleanup_v2(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() + + +def test_supabase_transaction_failure_rolls_back_and_has_no_background_repair() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, v1_service = _v1_service() + model_id = None + try: + model_id, parameter_name = _seed_catalog(v2_sessions) + creation = _create_v1(v1_service, parameter_name) + + class FailingMirror: + def mirror_legacy_policy(self, snapshot): + with v2_sessions.begin() as session: + persist_legacy_policy(session, snapshot) + raise OperationalError( + "forced transaction failure", + {}, + RuntimeError("forced"), + ) + + with pytest.raises(PolicyMirrorUnavailableError): + mirror_policy_after_commit( + creation.snapshot, + mirror_factory=FailingMirror, + ) + + with v2_sessions() as session: + assert session.scalar(select(func.count()).select_from(Policy)) == 0 + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 0 + ) + with v1_engine.connect() as connection: + assert connection.scalar(select(func.count()).select_from(V1Policy)) == 1 + + # No process is scheduled to change this state. Only an explicit retry + # invokes the mirror and creates the missing destination rows. + with v2_sessions() as session: + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 0 + ) + result = mirror_policy_after_commit( + creation.snapshot, + mirror_factory=lambda: V2PolicyService(v2_sessions), + ) + assert result.mapping_created is True + finally: + _cleanup_v2(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() diff --git a/tests/integration/test_v1_user_policy_dual_write.py b/tests/integration/test_v1_user_policy_dual_write.py new file mode 100644 index 000000000..cc7335a0a --- /dev/null +++ b/tests/integration/test_v1_user_policy_dual_write.py @@ -0,0 +1,386 @@ +"""Cross-database tests for immediate v1 saved-policy association mirroring.""" + +from __future__ import annotations + +import os + +import pytest +from sqlalchemy import create_engine, delete, func, select +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v1_models import UserPolicyMirrorEvent +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + LegacyUserPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + UserPolicy, +) +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.data.v2.user_policies.service import V2UserPolicyService +from policyengine_api.services.policy_service import PolicyService +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorUnavailableError, + mirror_pending_user_policy_events_after_commit, + mirror_user_policy_after_commit, +) +from policyengine_api.services.user_policy_service import UserPolicyService + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get( + V2_ALEMBIC_DISPOSABLE_TEST, + "", + ), + } + ) + if not settings.disposable_test: + pytest.fail("saved-policy cross-database tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +def _v1_services(): + engine = create_engine("sqlite://") + with engine.begin() as connection: + connection.exec_driver_sql( + """ + CREATE TABLE policy ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + country_id VARCHAR(3) NOT NULL, + label VARCHAR(255), + api_version VARCHAR(10) NOT NULL, + policy_json JSON NOT NULL, + policy_hash VARCHAR(255) NOT NULL + ) + """ + ) + connection.exec_driver_sql( + """ + CREATE TABLE user_policies ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + country_id VARCHAR(3) NOT NULL, + reform_id INTEGER NOT NULL, + reform_label VARCHAR(255), + baseline_id INTEGER NOT NULL, + baseline_label VARCHAR(255), + user_id VARCHAR(255) NOT NULL, + year VARCHAR(32) NOT NULL, + geography VARCHAR(255) NOT NULL, + dataset VARCHAR(255), + number_of_provisions INTEGER NOT NULL, + api_version VARCHAR(32) NOT NULL, + added_date BIGINT NOT NULL, + updated_date BIGINT NOT NULL, + budgetary_impact VARCHAR(255), + type VARCHAR(255), + mirror_revision BIGINT NOT NULL DEFAULT 0 + ) + """ + ) + UserPolicyMirrorEvent.__table__.create(connection) + sessions = sessionmaker(engine, expire_on_commit=False) + return engine, PolicyService(sessions), UserPolicyService(sessions), sessions + + +def _seed_catalog(sessions): + with sessions.begin() as session: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version=POLICYENGINE_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.phase10.cross_database_saved_rate", + tax_benefit_model_version=version, + ) + session.add(parameter) + session.flush() + return model.id, parameter.name + + +def _saved_values(reform_id: int, **changes): + values = { + "country_id": "us", + "reform_id": reform_id, + "reform_label": "Reform", + "baseline_id": 0, + "baseline_label": "Current law", + "user_id": "auth0|cross-database", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + values.update(changes) + return values + + +def _cleanup(engine, model_id) -> None: + if model_id is None: + return + with engine.begin() as connection: + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + association_ids = select(UserPolicy.id).where( + UserPolicy.policy_id.in_(policy_ids) + ) + connection.execute( + delete(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.user_policy_id.in_(association_ids) + ) + ) + connection.execute( + delete(UserPolicy).where(UserPolicy.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id.in_(policy_ids) + ) + ) + connection.execute( + delete(ParameterValue).where(ParameterValue.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def test_create_update_and_v1_only_change_mirror_one_association() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, policy_service, saved_service, _v1_sessions = _v1_services() + model_id = None + try: + model_id, parameter_name = _seed_catalog(v2_sessions) + reform = policy_service.set_policy( + "us", + "Legacy reform label", + {parameter_name: {"2026": 0.2}}, + ) + created = saved_service.create_or_get_user_policy( + _saved_values(reform.policy_id), + record_mirror_event=True, + ) + mirror_service = V2UserPolicyService(v2_sessions) + first = mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + renamed = saved_service.update_user_policy( + "us", + created.user_policy.id, + {"reform_label": "Renamed", "updated_date": 3}, + record_mirror_event=True, + ) + rename_result = mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=renamed.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + v1_only = saved_service.update_user_policy( + "us", + created.user_policy.id, + {"year": "2027", "updated_date": 4}, + record_mirror_event=True, + ) + v1_only_result = mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=v1_only.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert first.association_id == rename_result.association_id + assert first.association_id == v1_only_result.association_id + assert rename_result.association_updated is True + assert v1_only_result.association_updated is False + with v2_sessions() as session: + association = session.get(UserPolicy, first.association_id) + assert association.name == "Renamed" + assert association.description is None + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 1 + ) + mapping = session.scalar(select(LegacyUserPolicyMapping)) + assert mapping.last_applied_source_revision == 3 + finally: + _cleanup(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() + + +def test_failure_after_cloud_commit_and_identical_create_retry_are_idempotent() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, policy_service, saved_service, v1_sessions = _v1_services() + model_id = None + parameter_name = "gov.phase10.cross_database_saved_rate" + try: + reform = policy_service.set_policy( + "us", + "Legacy reform label", + {parameter_name: {"2026": 0.2}}, + ) + created = saved_service.create_or_get_user_policy( + _saved_values(reform.policy_id), + record_mirror_event=True, + ) + mirror_service = V2UserPolicyService(v2_sessions) + + with pytest.raises(UserPolicyMirrorUnavailableError): + mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + with v1_sessions() as session: + assert ( + session.scalar( + select(func.count()).select_from(created.user_policy.__class__) + ) + == 1 + ) + event = session.scalar(select(UserPolicyMirrorEvent)) + assert event.processed_at is None + with v2_sessions() as session: + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 0 + ) + + model_id, _parameter_name = _seed_catalog(v2_sessions) + retry = saved_service.create_or_get_user_policy( + _saved_values(reform.policy_id, number_of_provisions=99), + record_mirror_event=True, + ) + result = mirror_pending_user_policy_events_after_commit( + "us", + retry.user_policy.id, + through_revision=retry.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert retry.created is False + assert retry.user_policy.id == created.user_policy.id + assert result.association_created is False + with v1_sessions() as session: + events = session.scalars(select(UserPolicyMirrorEvent)).all() + assert [event.source_revision for event in events] == [1, 2] + assert all(event.processed_at is not None for event in events) + with v2_sessions() as session: + mapping = session.scalar(select(LegacyUserPolicyMapping)) + assert mapping.last_applied_source_revision == 2 + finally: + _cleanup(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() + + +def test_destination_commit_replays_when_source_processing_marker_is_missing() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, policy_service, saved_service, v1_sessions = _v1_services() + model_id = None + try: + model_id, parameter_name = _seed_catalog(v2_sessions) + reform = policy_service.set_policy( + "us", + "Legacy reform label", + {parameter_name: {"2026": 0.2}}, + ) + created = saved_service.create_or_get_user_policy( + _saved_values(reform.policy_id), + record_mirror_event=True, + ) + mirror_service = V2UserPolicyService(v2_sessions) + + committed = mirror_user_policy_after_commit( + created.snapshot, + reform.snapshot, + source_revision=created.mirror_revision, + mirror_factory=lambda: mirror_service, + ) + with v1_sessions() as session: + event = session.scalar(select(UserPolicyMirrorEvent)) + assert event.processed_at is None + + replayed = mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert replayed.association_id == committed.association_id + assert replayed.association_created is False + with v1_sessions() as session: + event = session.scalar(select(UserPolicyMirrorEvent)) + assert event.processed_at is not None + with v2_sessions() as session: + assert session.scalar(select(func.count()).select_from(UserPolicy)) == 1 + mapping = session.scalar(select(LegacyUserPolicyMapping)) + assert mapping.last_applied_source_revision == 1 + finally: + _cleanup(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() diff --git a/tests/integration/test_v2_policy_persistence.py b/tests/integration/test_v2_policy_persistence.py new file mode 100644 index 000000000..691ab678f --- /dev/null +++ b/tests/integration/test_v2_policy_persistence.py @@ -0,0 +1,455 @@ +"""PostgreSQL transaction tests for immutable policy persistence.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import os +from threading import Barrier +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import create_engine, delete, func, select +from sqlalchemy.exc import IntegrityError +from sqlmodel import Session + +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.data.v2.policies.canonicalization import ( + CanonicalPolicyContent, + canonical_policy_document, + canonicalize_policy, +) +from policyengine_api.data.v2.policies.persistence import ( + PolicyContentHashCollisionError, + persist_resolved_policy, +) +from policyengine_api.data.v2.policies.legacy import ( + LegacyPolicyMappingIntegrityError, + LegacyPolicySnapshot, + persist_legacy_policy, +) +from policyengine_api.data.v2.policies.schemas import ResolvedPolicyCreateCommand +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get( + V2_ALEMBIC_DISPOSABLE_TEST, + "", + ), + } + ) + if not settings.disposable_test: + pytest.fail("policy persistence tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +def _catalog(session: Session, *, country_id: str | None = None): + unique = uuid4().hex + model = TaxBenefitModel( + name=( + f"policyengine-{country_id}" + if country_id is not None + else f"phase10-policy-{unique[:8]}" + ) + ) + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name=f"gov.phase10.{unique}", + tax_benefit_model_version=version, + ) + session.add(parameter) + session.flush() + return model, version, parameter + + +def _command( + model_id: UUID, + version_id: UUID, + parameter_id: UUID, + *, + value: object = 0.2, +) -> ResolvedPolicyCreateCommand: + return ResolvedPolicyCreateCommand( + country_id="us", + tax_benefit_model_id=model_id, + tax_benefit_model_version_id=version_id, + policyengine_version="5.2.0", + parameter_values=[ + { + "parameter_id": parameter_id, + "value": value, + "start_date": "2026-01-01T00:00:00Z", + } + ], + ) + + +def _cleanup(engine, model_id: UUID) -> None: + with engine.begin() as connection: + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + connection.execute( + delete(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id.in_(policy_ids) + ) + ) + connection.execute( + delete(ParameterValue).where(ParameterValue.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def test_equivalent_create_returns_one_policy_and_one_child_set() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session) + model_id = model.id + command = _command(model.id, version.id, parameter.id) + first = persist_resolved_policy(session, command) + + with Session(engine) as session, session.begin(): + second = persist_resolved_policy(session, command) + + assert first.created is True + assert second.created is False + assert second.policy_id == first.policy_id + + with Session(engine) as session: + policy_count = session.scalar( + select(func.count()) + .select_from(Policy) + .where(Policy.id == first.policy_id) + ) + value_count = session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id == first.policy_id) + ) + assert (policy_count, value_count) == (1, 1) + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_equal_hash_with_different_canonical_bytes_is_an_integrity_error() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session) + model_id = model.id + version_id = version.id + parameter_id = parameter.id + original = _command(model_id, version_id, parameter_id, value=0.2) + stored = canonicalize_policy(original) + persist_resolved_policy(session, original) + + changed = _command(model_id, version_id, parameter_id, value=0.3) + + def simulated_collision( + command: ResolvedPolicyCreateCommand, + ) -> CanonicalPolicyContent: + return CanonicalPolicyContent( + version=stored.version, + document=canonical_policy_document(command), + content_hash=stored.content_hash, + ) + + with Session(engine) as session, session.begin(): + with pytest.raises(PolicyContentHashCollisionError): + persist_resolved_policy( + session, + changed, + canonicalizer=simulated_collision, + ) + + with Session(engine) as session: + values = session.scalars( + select(ParameterValue.value_json) + .join( + Policy, + Policy.id == ParameterValue.policy_id, + ) + .where(Policy.tax_benefit_model_id == model_id) + ).all() + assert values == [0.2] + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_legacy_policy_mapping_is_many_to_one_and_retry_safe() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session, country_id="us") + model_id = model.id + first = LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=301, + label="First label", + api_version="1.0.0", + policy_json={parameter.name: {"2026": 0.2}}, + source_policy_hash="first-legacy-hash", + ) + second = LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=302, + label="Second label", + api_version="1.0.0", + policy_json={parameter.name: {"2026": 0.2}}, + source_policy_hash="second-legacy-hash", + ) + first_result = persist_legacy_policy( + session, + first, + running_policyengine_version=version.version, + country_package_versions={"us": "1.0.0"}, + ) + second_result = persist_legacy_policy( + session, + second, + running_policyengine_version=version.version, + country_package_versions={"us": "1.0.0"}, + ) + + with Session(engine) as session, session.begin(): + retry = persist_legacy_policy( + session, + first, + running_policyengine_version="5.2.0", + country_package_versions={"us": "1.0.0"}, + ) + + assert first_result.policy_id == second_result.policy_id == retry.policy_id + assert first_result.policy_created is True + assert second_result.policy_created is False + assert retry == type(retry)( + policy_id=first_result.policy_id, + policy_created=False, + mapping_created=False, + ) + with Session(engine) as session: + mappings = session.scalars( + select(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id == first_result.policy_id + ) + ).all() + assert {mapping.legacy_policy_id for mapping in mappings} == {301, 302} + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_changed_hash_for_one_legacy_identity_rolls_back_without_mutation() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session, country_id="us") + model_id = model.id + parameter_name = parameter.name + snapshot = LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=401, + api_version="1.0.0", + policy_json={parameter_name: {"2026": 0.2}}, + source_policy_hash="committed-source-hash", + ) + result = persist_legacy_policy( + session, + snapshot, + running_policyengine_version=version.version, + country_package_versions={"us": "1.0.0"}, + ) + + changed = snapshot.model_copy( + update={ + "source_policy_hash": "different-source-hash", + "policy_json": {parameter_name: {"2026": 0.3}}, + } + ) + with pytest.raises(LegacyPolicyMappingIntegrityError, match="different"): + with Session(engine) as session, session.begin(): + persist_legacy_policy( + session, + changed, + running_policyengine_version="5.2.0", + country_package_versions={"us": "1.0.0"}, + ) + + with Session(engine) as session: + mapping = session.scalar( + select(LegacyPolicyMapping).where( + LegacyPolicyMapping.legacy_policy_id == 401 + ) + ) + policy_count = session.scalar( + select(func.count()) + .select_from(Policy) + .where(Policy.tax_benefit_model_id == model_id) + ) + assert mapping is not None + assert mapping.policy_id == result.policy_id + assert mapping.source_policy_hash == "committed-source-hash" + assert policy_count == 1 + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_concurrent_equivalent_creates_return_one_policy_uuid() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session) + model_id = model.id + command = _command(model.id, version.id, parameter.id) + + barrier = Barrier(2, timeout=10) + + def create(): + with Session(engine) as session, session.begin(): + barrier.wait() + return persist_resolved_policy(session, command) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(create) for _ in range(2)] + results = [future.result(timeout=15) for future in futures] + + assert {result.policy_id for result in results} == {results[0].policy_id} + assert sorted(result.created for result in results) == [False, True] + with Session(engine) as session: + policy_count = session.scalar( + select(func.count()) + .select_from(Policy) + .where(Policy.tax_benefit_model_id == model_id) + ) + value_count = session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id == results[0].policy_id) + ) + assert (policy_count, value_count) == (1, 1) + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_empty_and_distinct_policy_content_persist_independently() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session) + model_id = model.id + empty = ResolvedPolicyCreateCommand( + country_id="us", + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=version.id, + policyengine_version=version.version, + parameter_values=[], + ) + first = persist_resolved_policy(session, empty) + second = persist_resolved_policy( + session, + _command(model.id, version.id, parameter.id, value=1), + ) + third = persist_resolved_policy( + session, + _command(model.id, version.id, parameter.id, value=2), + ) + + assert len({first.policy_id, second.policy_id, third.policy_id}) == 3 + with Session(engine) as session: + empty_value_count = session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id == first.policy_id) + ) + assert empty_value_count == 0 + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_child_insert_failure_rolls_back_the_policy_and_all_values() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, _parameter = _catalog(session) + model_id = model.id + invalid = _command(model.id, version.id, uuid4()) + content_hash = canonicalize_policy(invalid).content_hash + + with pytest.raises(IntegrityError): + with Session(engine) as session, session.begin(): + persist_resolved_policy(session, invalid) + + with Session(engine) as session: + policy_count = session.scalar( + select(func.count()) + .select_from(Policy) + .where(Policy.content_hash == content_hash) + ) + owned_value_count = session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id.is_not(None)) + ) + assert policy_count == 0 + assert owned_value_count == 0 + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() diff --git a/tests/integration/test_v2_user_policy_mirroring.py b/tests/integration/test_v2_user_policy_mirroring.py new file mode 100644 index 000000000..582e8f2c1 --- /dev/null +++ b/tests/integration/test_v2_user_policy_mirroring.py @@ -0,0 +1,355 @@ +"""PostgreSQL transaction tests for legacy saved-policy projection.""" + +from __future__ import annotations + +import os + +import pytest +from sqlalchemy import create_engine, delete, func, select +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + LegacyUserPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + UserPolicy, +) +from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.data.v2.user_policies.legacy import ( + LegacyUserPolicySnapshot, + fingerprint_legacy_user_policy, + persist_legacy_user_policy, +) + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get( + V2_ALEMBIC_DISPOSABLE_TEST, + "", + ), + } + ) + if not settings.disposable_test: + pytest.fail("saved-policy mirror tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +def _seed_catalog(sessions): + with sessions.begin() as session: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version=POLICYENGINE_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.phase10.saved_policy_rate", + tax_benefit_model_version=version, + ) + session.add(parameter) + session.flush() + return model.id, parameter.name + + +def _reform(parameter_name: str, *, legacy_id: int, source_hash: str): + return LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=legacy_id, + label="Ignored legacy label", + api_version=COUNTRY_PACKAGE_VERSIONS["us"], + policy_json={parameter_name: {"2026": 0.2}}, + source_policy_hash=source_hash, + ) + + +def _saved(*, legacy_id: int, reform_id: int, reform_label="Reform", **changes): + values = { + "country_id": "us", + "legacy_user_policy_id": legacy_id, + "reform_id": reform_id, + "reform_label": reform_label, + "baseline_id": 1, + "baseline_label": "Current law", + "user_id": "auth0|one", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + values.update(changes) + return LegacyUserPolicySnapshot.model_validate(values) + + +def _cleanup(engine, model_id) -> None: + if model_id is None: + return + with engine.begin() as connection: + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + association_ids = select(UserPolicy.id).where( + UserPolicy.policy_id.in_(policy_ids) + ) + connection.execute( + delete(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.user_policy_id.in_(association_ids) + ) + ) + connection.execute( + delete(UserPolicy).where(UserPolicy.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id.in_(policy_ids) + ) + ) + connection.execute( + delete(ParameterValue).where(ParameterValue.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def test_distinct_saved_rows_share_policy_and_preserve_nullable_names() -> None: + engine = create_engine(_disposable_url()) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + model_id = None + try: + model_id, parameter_name = _seed_catalog(sessions) + first_reform = _reform(parameter_name, legacy_id=101, source_hash="first") + second_reform = _reform(parameter_name, legacy_id=102, source_hash="second") + first_saved = _saved(legacy_id=201, reform_id=101) + second_saved = _saved( + legacy_id=202, + reform_id=102, + reform_label=None, + ) + + with sessions.begin() as session: + first = persist_legacy_user_policy( + session, + first_saved, + first_reform, + source_revision=1, + ) + second = persist_legacy_user_policy( + session, + second_saved, + second_reform, + source_revision=1, + ) + with sessions.begin() as session: + retry = persist_legacy_user_policy( + session, + first_saved, + first_reform, + source_revision=1, + ) + + assert first.policy_id == second.policy_id + assert first.association_id != second.association_id + assert retry.association_id == first.association_id + assert retry.association_created is False + with sessions() as session: + associations = session.scalars( + select(UserPolicy).order_by(UserPolicy.created_at, UserPolicy.id) + ).all() + assert {association.name for association in associations} == { + "Reform", + None, + } + assert all(association.description is None for association in associations) + assert session.scalar(select(func.count()).select_from(Policy)) == 1 + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 2 + ) + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 2 + ) + finally: + _cleanup(engine, model_id) + engine.dispose() + + +def test_label_and_v1_only_updates_advance_the_complete_row_fingerprint() -> None: + engine = create_engine(_disposable_url()) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + model_id = None + try: + model_id, parameter_name = _seed_catalog(sessions) + reform = _reform(parameter_name, legacy_id=301, source_hash="reform") + original = _saved(legacy_id=401, reform_id=301) + with sessions.begin() as session: + created = persist_legacy_user_policy( + session, + original, + reform, + source_revision=1, + ) + + renamed = original.model_copy( + update={"reform_label": "Renamed", "updated_date": 3} + ) + with sessions.begin() as session: + association = session.get(UserPolicy, created.association_id) + association.description = "Native description" + with sessions.begin() as session: + rename_result = persist_legacy_user_policy( + session, + renamed, + reform, + source_revision=2, + changed_fields=frozenset({"reform_label", "updated_date"}), + ) + with sessions() as session: + after_rename = session.get(UserPolicy, created.association_id) + mapping = session.scalar( + select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.user_policy_id == created.association_id + ) + ) + assert after_rename.name == "Renamed" + assert after_rename.description == "Native description" + assert mapping.fingerprint_sha256 == fingerprint_legacy_user_policy(renamed) + + v1_only = renamed.model_copy(update={"year": "2027", "updated_date": 4}) + with sessions.begin() as session: + association = session.get(UserPolicy, created.association_id) + association.name = "Native name" + with sessions() as session: + native_timestamp = session.get( + UserPolicy, + created.association_id, + ).updated_at + with sessions.begin() as session: + v1_only_result = persist_legacy_user_policy( + session, + v1_only, + reform, + source_revision=3, + changed_fields=frozenset({"year", "updated_date"}), + ) + with sessions() as session: + association = session.get(UserPolicy, created.association_id) + mapping = session.scalar( + select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.user_policy_id == created.association_id + ) + ) + + assert rename_result.association_updated is True + assert v1_only_result.association_updated is False + assert association.name == "Native name" + assert association.description == "Native description" + assert association.updated_at == native_timestamp + assert mapping.fingerprint_sha256 == fingerprint_legacy_user_policy(v1_only) + finally: + _cleanup(engine, model_id) + engine.dispose() + + +def test_complete_transaction_rolls_back_and_native_delete_is_isolated() -> None: + engine = create_engine(_disposable_url()) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + model_id = None + try: + model_id, parameter_name = _seed_catalog(sessions) + reform = _reform(parameter_name, legacy_id=501, source_hash="rollback") + saved = _saved(legacy_id=601, reform_id=501) + with pytest.raises(RuntimeError, match="forced rollback"): + with sessions.begin() as session: + persist_legacy_user_policy( + session, + saved, + reform, + source_revision=1, + ) + raise RuntimeError("forced rollback") + + with sessions() as session: + assert session.scalar(select(func.count()).select_from(Policy)) == 0 + assert session.scalar(select(func.count()).select_from(UserPolicy)) == 0 + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 0 + ) + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 0 + ) + + with sessions.begin() as session: + created = persist_legacy_user_policy( + session, + saved, + reform, + source_revision=1, + ) + with sessions.begin() as session: + association = session.get(UserPolicy, created.association_id) + session.delete(association) + + with sessions() as session: + assert session.get(UserPolicy, created.association_id) is None + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 0 + ) + assert session.get(Policy, created.policy_id) is not None + assert ( + session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id == created.policy_id) + ) + == 1 + ) + finally: + _cleanup(engine, model_id) + engine.dispose() diff --git a/tests/unit/data/test_v1_models.py b/tests/unit/data/test_v1_models.py index d61c5cacb..496e4adea 100644 --- a/tests/unit/data/test_v1_models.py +++ b/tests/unit/data/test_v1_models.py @@ -17,6 +17,7 @@ "simulation_runs", "simulations", "user_policies", + "user_policy_mirror_events", "user_profiles", } @@ -49,6 +50,14 @@ def test_v1_composite_and_unique_keys_match_legacy_contract(): "country_id", ] assert V1Base.metadata.tables["user_profiles"].c.auth0_id.unique + assert V1Base.metadata.tables["user_policies"].c.mirror_revision.default.arg == 0 + mirror_events = V1Base.metadata.tables["user_policy_mirror_events"] + assert mirror_events.c.id.autoincrement is True + assert { + tuple(column.name for column in constraint.columns) + for constraint in mirror_events.constraints + if constraint.__class__.__name__ == "UniqueConstraint" + } == {("country_id", "legacy_user_policy_id", "source_revision")} assert ( V1Base.metadata.tables[ "legacy_report_output_aliases" diff --git a/tests/unit/routes/test_migration_context_logging.py b/tests/unit/routes/test_migration_context_logging.py index 1a487dfeb..1aa6609fa 100644 --- a/tests/unit/routes/test_migration_context_logging.py +++ b/tests/unit/routes/test_migration_context_logging.py @@ -273,6 +273,50 @@ def test_v2_metadata_resource_logs_its_actual_supabase_read_source(monkeypatch): assert migration_context["db_read"] == "supabase" +def test_v2_policy_resources_log_actual_supabase_read_and_write_sources( + monkeypatch, +): + monkeypatch.setenv("DB_READ_POLICY", "invalid-for-native-v2") + monkeypatch.setenv("DB_WRITE_POLICY", "invalid-for-native-v2") + + with patch("policyengine_api.migration_logging.logger") as mock_logger: + log_migration_request( + request_id="request-read", + method="GET", + path="/v2/policies", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + log_migration_request( + request_id="request-write", + method="PATCH", + path="/v2/user-policies/00000000-0000-0000-0000-000000000001", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + + read_context = mock_logger.log_struct.call_args_list[0].args[0]["migration"] + write_context = mock_logger.log_struct.call_args_list[1].args[0]["migration"] + assert read_context == { + **read_context, + "route_group": "policy", + "route_impl": "fastapi_native", + "db_write": None, + "db_read": "supabase", + } + assert write_context == { + **write_context, + "route_group": "policy", + "route_impl": "fastapi_native", + "db_write": "supabase", + "db_read": None, + } + + def test_native_route_failure_logs_country_and_actual_implementation(): dependencies = NativeRouteDependencies( readiness_probe=lambda: True, diff --git a/tests/unit/routes/test_policy_dual_write_routes.py b/tests/unit/routes/test_policy_dual_write_routes.py new file mode 100644 index 000000000..d47667f6f --- /dev/null +++ b/tests/unit/routes/test_policy_dual_write_routes.py @@ -0,0 +1,207 @@ +"""V1 policy route tests for immediate post-commit v2 mirroring.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +from flask import Flask + +from policyengine_api.data.v1_models import Policy +from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot +from policyengine_api.routes.policy_routes import policy_bp +from policyengine_api.services.policy_mirroring import PolicyMirrorUnavailableError +from policyengine_api.services.policy_service import PolicySetResult + + +def _client(): + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(policy_bp) + return app.test_client() + + +def _snapshot() -> LegacyPolicySnapshot: + return LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=42, + label="Legacy label", + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + source_policy_hash="legacy/base64+hash=", + ) + + +def _creation(*, existing: bool = False) -> PolicySetResult: + return PolicySetResult( + policy_id=42, + message="Policy already exists" if existing else "Policy created", + is_existing_policy=existing, + snapshot=_snapshot(), + ) + + +def _body() -> dict[str, object]: + return { + "label": "Legacy label", + "data": {"gov.example.rate": {"2026": 0.2}}, + } + + +def test_cloud_sql_mode_preserves_v1_create_response_without_mirroring( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy", + return_value=_creation(), + ) as set_policy, + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit" + ) as mirror, + ): + response = _client().post("/us/policy", json=_body()) + + assert response.status_code == 201 + assert response.json == { + "status": "ok", + "message": "Policy created", + "result": {"policy_id": 42}, + } + set_policy.assert_called_once() + mirror.assert_not_called() + + +def test_dual_write_mirrors_new_and_existing_rows_before_success( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + for existing, expected_status in ((False, 201), (True, 200)): + events: list[str] = [] + service = MagicMock( + side_effect=lambda *_args: ( + events.append("cloud_sql") or _creation(existing=existing) + ) + ) + mirror = MagicMock(side_effect=lambda _snapshot: events.append("supabase")) + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy", + service, + ), + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit", + mirror, + ), + ): + response = _client().post("/us/policy", json=_body()) + + assert response.status_code == expected_status + assert response.json["result"] == {"policy_id": 42} + assert "v2" not in response.json["result"] + assert events == ["cloud_sql", "supabase"] + mirror.assert_called_once_with(_snapshot()) + + +def test_mirror_failure_returns_503_and_identical_retry_completes( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + creation = _creation(existing=True) + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy", + return_value=creation, + ) as set_policy, + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit", + side_effect=[ + PolicyMirrorUnavailableError("database credential secret"), + MagicMock(), + ], + ) as mirror, + ): + first = _client().post("/us/policy", json=_body()) + retry = _client().post("/us/policy", json=_body()) + + assert first.status_code == 503 + assert first.json == { + "status": "error", + "message": "V2 policy mirroring is unavailable; retry the same request.", + } + assert "secret" not in first.text + assert retry.status_code == 200 + assert retry.json["result"] == {"policy_id": 42} + assert set_policy.call_count == 2 + assert mirror.call_count == 2 + + +def test_supabase_only_v1_write_selection_is_rejected_before_cloud_sql( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "supabase") + with patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy" + ) as set_policy: + response = _client().post("/us/policy", json=_body()) + + assert response.status_code == 503 + assert response.json["status"] == "error" + set_policy.assert_not_called() + + +def test_v1_policy_reads_require_cloud_sql_and_never_invoke_v2( + monkeypatch, +) -> None: + policy = Policy( + id=42, + country_id="us", + label="Legacy label", + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + policy_hash="legacy/base64+hash=", + ) + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy", + return_value=policy, + ) as get_policy, + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit" + ) as mirror, + ): + response = _client().get("/us/policy/42") + + assert response.status_code == 200 + assert json.loads(response.text)["result"]["id"] == 42 + get_policy.assert_called_once_with("us", 42) + mirror.assert_not_called() + + monkeypatch.setenv("DB_READ_POLICY", "read_compare") + with patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy" + ) as invalid_get: + invalid = _client().get("/us/policy/42") + + assert invalid.status_code == 503 + invalid_get.assert_not_called() + + +def test_policy_search_failure_does_not_expose_exception_details( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + with patch( + "policyengine_api.routes.policy_routes.policy_service.search_policies", + side_effect=RuntimeError("database-credential-secret"), + ): + response = _client().get("/us/policies?query=example") + + assert response.status_code == 500 + assert response.json == { + "status": "error", + "message": "Internal server error; please try again later.", + } + assert "database-credential-secret" not in response.text diff --git a/tests/unit/routes/test_user_policy_dual_write_routes.py b/tests/unit/routes/test_user_policy_dual_write_routes.py new file mode 100644 index 000000000..881266845 --- /dev/null +++ b/tests/unit/routes/test_user_policy_dual_write_routes.py @@ -0,0 +1,447 @@ +"""V1 saved-policy route tests for immediate v2 association mirroring.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +from flask import Flask +import pytest +from sqlalchemy.exc import ( + IntegrityError, + OperationalError, + SQLAlchemyError, + TimeoutError as SQLAlchemyTimeoutError, +) + +from policyengine_api.data.v1_models import UserPolicy +from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot +from policyengine_api.data.v2.user_policies.legacy import LegacyUserPolicySnapshot +from policyengine_api.routes.policy_routes import policy_bp +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorUnavailableError, +) +from policyengine_api.services.user_policy_service import ( + UserPolicyCreateResult, + UserPolicyPersistenceError, + UserPolicyUpdateResult, +) + + +def _client(): + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(policy_bp) + return app.test_client() + + +def _row(*, reform_label: str | None = "Reform", year: str = "2026"): + return UserPolicy( + id=10, + country_id="us", + reform_id=2, + reform_label=reform_label, + baseline_id=1, + baseline_label="Current law", + user_id="auth0|one", + year=year, + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + + +def _snapshot(*, reform_label: str | None = "Reform", year: str = "2026"): + return LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=10, + reform_id=2, + reform_label=reform_label, + baseline_id=1, + baseline_label="Current law", + user_id="auth0|one", + year=year, + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + + +def _reform_snapshot(): + return LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=2, + label="Ignored core label", + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + source_policy_hash="legacy/base64+hash=", + ) + + +def _creation(*, created=True, reform_label="Reform", mirror_revision=1): + return UserPolicyCreateResult( + user_policy=_row(reform_label=reform_label), + created=created, + snapshot=_snapshot(reform_label=reform_label), + mirror_revision=mirror_revision, + ) + + +def _body(*, reform_label="Reform") -> dict[str, object]: + return { + "reform_id": 2, + "reform_label": reform_label, + "baseline_id": 1, + "baseline_label": "Current law", + "user_id": "auth0|one", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + + +def test_cloud_sql_mode_preserves_create_without_association_mirror( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.create_or_get_user_policy", + return_value=_creation(), + ), + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as mirror, + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy_snapshot" + ) as get_reform, + ): + response = _client().post("/us/user-policy", json=_body()) + + assert response.status_code == 201 + assert response.json["result"]["id"] == 10 + assert "v2" not in response.json["result"] + mirror.assert_not_called() + get_reform.assert_not_called() + + +def test_dual_write_mirrors_new_existing_and_unlabeled_saved_policies( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + for created, label, expected_status in ( + (True, "Reform", 201), + (False, "Reform", 200), + (True, None, 201), + ): + creation = _creation(created=created, reform_label=label) + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.create_or_get_user_policy", + return_value=creation, + ), + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy_snapshot", + return_value=_reform_snapshot(), + ), + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as mirror, + ): + response = _client().post( + "/us/user-policy", + json=_body(reform_label=label), + ) + + assert response.status_code == expected_status + assert response.json["result"]["id"] == 10 + assert "v2" not in response.json["result"] + mirror.assert_called_once() + assert mirror.call_args.args == ("us", 10) + assert mirror.call_args.kwargs["through_revision"] == 1 + + +def test_saved_policy_mirror_failure_returns_503_and_retry_completes( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + creation = _creation(created=False) + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.create_or_get_user_policy", + return_value=creation, + ) as create, + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy_snapshot", + return_value=_reform_snapshot(), + ), + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit", + side_effect=[ + UserPolicyMirrorUnavailableError("database credential secret"), + MagicMock(), + ], + ) as mirror, + ): + first = _client().post("/us/user-policy", json=_body()) + retry = _client().post("/us/user-policy", json=_body()) + + assert first.status_code == 503 + assert first.json == { + "message": "V2 saved-policy mirroring is unavailable; retry the same request." + } + assert "secret" not in first.text + assert retry.status_code == 200 + assert retry.json["result"] == {"id": 10} + assert create.call_count == mirror.call_count == 2 + + +@pytest.mark.parametrize( + ( + "method", + "service_method", + "body", + "error", + "expected_status", + "expected_category", + "expected_operation", + ), + ( + ( + "post", + "create_or_get_user_policy", + _body(), + SQLAlchemyTimeoutError("credential=timeout-secret"), + 503, + "timeout", + "create", + ), + ( + "put", + "update_user_policy", + {"id": 10, "reform_label": "Renamed"}, + OperationalError( + "UPDATE user_policies SET secret=:secret", + {"secret": "bound-parameter-secret"}, + RuntimeError("driver-secret"), + ), + 503, + "unavailable", + "update", + ), + ( + "post", + "create_or_get_user_policy", + _body(), + IntegrityError( + "INSERT caller-private-data", + {"user_id": "caller-private-data"}, + RuntimeError("integrity-secret"), + ), + 500, + "integrity", + "create", + ), + ( + "put", + "update_user_policy", + {"id": 10, "year": "2027"}, + SQLAlchemyError("database-secret"), + 500, + "database", + "update", + ), + ( + "post", + "create_or_get_user_policy", + _body(), + RuntimeError("unexpected-secret"), + 500, + "unexpected", + "create", + ), + ( + "put", + "update_user_policy", + {"id": "caller-id-secret", "year": "2027"}, + RuntimeError("unexpected-secret"), + 500, + "unexpected", + "update", + ), + ), +) +def test_saved_policy_persistence_failures_use_allowlisted_records( + monkeypatch, + method, + service_method, + body, + error, + expected_status, + expected_category, + expected_operation, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + with ( + patch( + f"policyengine_api.routes.policy_routes.user_policy_service.{service_method}", + side_effect=UserPolicyPersistenceError.from_exception(error), + ), + patch( + "policyengine_api.routes.policy_routes.current_request_id", + return_value="request-123", + ), + patch("policyengine_api.routes.policy_routes.logger.log_struct") as log_struct, + ): + response = getattr(_client(), method)("/us/user-policy", json=body) + + expected_message = ( + "Policy database is temporarily unavailable; please try again later." + if expected_status == 503 + else "Internal database error; please try again later." + ) + assert response.status_code == expected_status + assert response.json == {"message": expected_message} + + log_struct.assert_called_once() + assert log_struct.call_args.kwargs == {"severity": "ERROR"} + payload = log_struct.call_args.args[0] + assert set(payload) == { + "message", + "metric_name", + "metric_value", + "resource", + "operation", + "database_source", + "configured_write_source", + "country_id", + "legacy_user_policy_id", + "request_id", + "outcome", + "failure_category", + "http_status", + "duration_ms", + } + assert payload["operation"] == expected_operation + assert payload["failure_category"] == expected_category + assert payload["http_status"] == expected_status + assert payload["request_id"] == "request-123" + supplied_id = body.get("id") + expected_logged_id = ( + supplied_id + if isinstance(supplied_id, int) + and not isinstance(supplied_id, bool) + and 0 <= supplied_id <= 2_147_483_647 + else None + ) + assert payload["legacy_user_policy_id"] == expected_logged_id + + serialized_record = json.dumps(payload, sort_keys=True) + for private_value in ( + "timeout-secret", + "bound-parameter-secret", + "driver-secret", + "caller-private-data", + "integrity-secret", + "database-secret", + "unexpected-secret", + "caller-id-secret", + ): + assert private_value not in response.text + assert private_value not in serialized_record + + +def test_update_mirrors_projected_and_v1_only_changes_before_success( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + for payload, snapshot in ( + ({"reform_label": "Renamed"}, _snapshot(reform_label="Renamed")), + ({"year": "2027"}, _snapshot(year="2027")), + ): + update = UserPolicyUpdateResult( + user_policy=_row( + reform_label=snapshot.reform_label, + year=snapshot.year, + ), + snapshot=snapshot, + changed_fields=frozenset(payload), + mirror_revision=1, + ) + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.update_user_policy", + return_value=update, + ), + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy_snapshot", + return_value=_reform_snapshot(), + ), + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as mirror, + ): + response = _client().put( + "/us/user-policy", + json={"id": 10, **payload}, + ) + + assert response.status_code == 200 + assert response.json["result"] == {"id": 10} + assert mirror.call_args.args == ("us", 10) + assert mirror.call_args.kwargs["through_revision"] == 1 + + +def test_saved_policy_reads_remain_cloud_sql_only(monkeypatch) -> None: + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.list_user_policies", + return_value=[_row()], + ) as list_rows, + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as mirror, + ): + response = _client().get("/us/user-policy/auth0|one") + + assert response.status_code == 200 + assert response.json["result"][0]["id"] == 10 + list_rows.assert_called_once_with("us", "auth0|one") + mirror.assert_not_called() + + monkeypatch.setenv("DB_READ_POLICY", "supabase") + with patch( + "policyengine_api.routes.policy_routes.user_policy_service.list_user_policies" + ) as invalid_list: + invalid = _client().get("/us/user-policy/auth0|one") + + assert invalid.status_code == 503 + invalid_list.assert_not_called() + + +def test_v1_saved_policy_delete_is_not_added() -> None: + response = _client().delete("/us/user-policy", json={"id": 10}) + + assert response.status_code == 405 diff --git a/tests/unit/services/test_policy_mirroring.py b/tests/unit/services/test_policy_mirroring.py new file mode 100644 index 000000000..908d54f1e --- /dev/null +++ b/tests/unit/services/test_policy_mirroring.py @@ -0,0 +1,103 @@ +"""Immediate v1 policy mirror orchestration and observability tests.""" + +from __future__ import annotations + +from uuid import UUID +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy.exc import OperationalError, TimeoutError + +from policyengine_api.data.v2.policies.legacy import ( + LegacyPolicyMappingIntegrityError, + LegacyPolicyPersistenceResult, + LegacyPolicySnapshot, +) +from policyengine_api.services.policy_mirroring import ( + PolicyMirrorUnavailableError, + mirror_policy_after_commit, +) + + +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") + + +def _snapshot() -> LegacyPolicySnapshot: + return LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=42, + label="Legacy label", + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + source_policy_hash="legacy/base64+hash=", + ) + + +def test_success_returns_destination_and_logs_metric_without_policy_content() -> None: + mirror = MagicMock() + mirror.mirror_legacy_policy.return_value = LegacyPolicyPersistenceResult( + policy_id=POLICY_ID, + policy_created=True, + mapping_created=True, + ) + + with patch("policyengine_api.services.policy_mirroring.logger") as logger: + result = mirror_policy_after_commit( + _snapshot(), + mirror_factory=lambda: mirror, + ) + + assert result.policy_id == POLICY_ID + payload = logger.log_struct.call_args.args[0] + assert payload == { + **payload, + "metric_name": "v1_policy_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": ["cloud_sql", "supabase"], + "country_id": "us", + "legacy_policy_id": 42, + "destination_policy_id": str(POLICY_ID), + "outcome": "ok", + "failure_category": None, + "policy_created": True, + "mapping_created": True, + } + rendered = repr(payload) + assert "gov.example.rate" not in rendered + assert "legacy/base64" not in rendered + + +@pytest.mark.parametrize( + ("error", "category"), + [ + ( + OperationalError("statement secret", {}, Exception("credential")), + "database", + ), + (TimeoutError("pool timeout"), "database"), + (LegacyPolicyMappingIntegrityError("changed source hash"), "integrity"), + (RuntimeError("policy content secret"), "unexpected"), + ], +) +def test_failure_is_secret_safe_and_records_cloud_sql_as_only_completed_write( + error: Exception, + category: str, +) -> None: + mirror = MagicMock() + mirror.mirror_legacy_policy.side_effect = error + + with ( + patch("policyengine_api.services.policy_mirroring.logger") as logger, + pytest.raises(PolicyMirrorUnavailableError, match="could not be mirrored"), + ): + mirror_policy_after_commit(_snapshot(), mirror_factory=lambda: mirror) + + payload = logger.log_struct.call_args.args[0] + assert payload["outcome"] == "error" + assert payload["failure_category"] == category + assert payload["actual_write_sources"] == ["cloud_sql"] + assert payload["destination_policy_id"] is None + assert "secret" not in repr(payload) + assert "credential" not in repr(payload) diff --git a/tests/unit/services/test_policy_service.py b/tests/unit/services/test_policy_service.py index a2b1c8062..121dbe28e 100644 --- a/tests/unit/services/test_policy_service.py +++ b/tests/unit/services/test_policy_service.py @@ -3,7 +3,7 @@ from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS from policyengine_api.data.v1_models import Policy -from policyengine_api.services.policy_service import PolicyService +from policyengine_api.services.policy_service import PolicyService, PolicySetResult from tests.fixtures.services.policy_service import valid_policy_data @@ -99,17 +99,27 @@ def test_set_policy_adds_mapped_entity(service, monkeypatch): lambda value: "new-hash", ) - policy_id, message, exists = service.set_policy( + result = service.set_policy( "US", "New policy", {"parameter": 1}, ) + policy_id, message, exists = result policy = service.get_policy("us", policy_id) assert policy.policy_json == {"parameter": 1} assert policy.api_version == COUNTRY_PACKAGE_VERSIONS["us"] assert message == "Policy created" assert exists is False + assert isinstance(result, PolicySetResult) + assert result.snapshot.model_dump() == { + "country_id": "us", + "legacy_policy_id": policy_id, + "label": "New policy", + "api_version": COUNTRY_PACKAGE_VERSIONS["us"], + "policy_json": {"parameter": 1}, + "source_policy_hash": "new-hash", + } def test_set_policy_returns_existing_mapped_entity( @@ -122,15 +132,18 @@ def test_set_policy_returns_existing_mapped_entity( lambda value: valid_policy_data["policy_hash"], ) - policy_id, message, exists = service.set_policy( + result = service.set_policy( "us", None, {}, ) + policy_id, message, exists = result assert policy_id == valid_policy_data["id"] assert message == "Policy already exists" assert exists is True + assert result.snapshot.legacy_policy_id == valid_policy_data["id"] + assert result.snapshot.source_policy_hash == valid_policy_data["policy_hash"] def test_set_policy_rejects_invalid_country(service): diff --git a/tests/unit/services/test_user_policy_mirroring.py b/tests/unit/services/test_user_policy_mirroring.py new file mode 100644 index 000000000..d64e4a3ae --- /dev/null +++ b/tests/unit/services/test_user_policy_mirroring.py @@ -0,0 +1,238 @@ +"""Saved-policy mirror observability and error conversion tests.""" + +from __future__ import annotations + +from uuid import UUID +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy import event as sqlalchemy_event, select +from sqlalchemy.exc import OperationalError, TimeoutError + +from policyengine_api.data.v1_models import UserPolicyMirrorEvent +from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot +from policyengine_api.data.v2.user_policies.legacy import ( + LegacyUserPolicyIntegrityError, + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, +) +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorUnavailableError, + mirror_pending_user_policy_events_after_commit, + mirror_user_policy_after_commit, +) +from policyengine_api.services.user_policy_service import UserPolicyService + + +ASSOCIATION_ID = UUID("00000000-0000-0000-0000-000000000060") +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") + + +def _saved() -> LegacyUserPolicySnapshot: + return LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=10, + reform_id=2, + reform_label="Reform", + baseline_id=1, + baseline_label="Current law", + user_id="auth0|one", + year="2026", + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + + +def _reform() -> LegacyPolicySnapshot: + return LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=2, + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + source_policy_hash="legacy/base64+hash=", + ) + + +def _saved_values() -> dict[str, object]: + return _saved().model_dump(exclude={"legacy_user_policy_id"}) + + +def test_success_logs_only_identifiers_outcomes_and_metric_fields() -> None: + mirror = MagicMock() + mirror.mirror_legacy_user_policy.return_value = LegacyUserPolicyPersistenceResult( + association_id=ASSOCIATION_ID, + policy_id=POLICY_ID, + association_created=True, + association_updated=False, + mapping_created=True, + ) + + with patch("policyengine_api.services.user_policy_mirroring.logger") as logger: + result = mirror_user_policy_after_commit( + _saved(), + _reform(), + source_revision=3, + changed_fields=frozenset({"reform_label"}), + mirror_factory=lambda: mirror, + ) + + assert result.association_id == ASSOCIATION_ID + mirror.mirror_legacy_user_policy.assert_called_once_with( + _saved(), + _reform(), + source_revision=3, + changed_fields=frozenset({"reform_label"}), + ) + payload = logger.log_struct.call_args.args[0] + assert payload["metric_name"] == "v1_user_policy_mirror_operations" + assert payload["configured_write_source"] == "dual_write" + assert payload["actual_write_sources"] == ["cloud_sql", "supabase"] + assert payload["legacy_user_policy_id"] == 10 + assert payload["destination_association_id"] == str(ASSOCIATION_ID) + assert payload["destination_policy_id"] == str(POLICY_ID) + rendered = repr(payload) + assert "gov.example.rate" not in rendered + assert "auth0|one" not in rendered + assert "Reform" not in rendered + + +@pytest.mark.parametrize( + ("error", "category"), + [ + ( + OperationalError("statement secret", {}, Exception("credential")), + "database", + ), + (TimeoutError("pool timeout"), "database"), + (LegacyUserPolicyIntegrityError("mapping conflict"), "integrity"), + (RuntimeError("caller data secret"), "unexpected"), + ], +) +def test_failure_is_secret_safe_and_reports_only_cloud_sql_completed( + error: Exception, + category: str, +) -> None: + mirror = MagicMock() + mirror.mirror_legacy_user_policy.side_effect = error + + with ( + patch("policyengine_api.services.user_policy_mirroring.logger") as logger, + pytest.raises(UserPolicyMirrorUnavailableError), + ): + mirror_user_policy_after_commit( + _saved(), + _reform(), + source_revision=3, + mirror_factory=lambda: mirror, + ) + + payload = logger.log_struct.call_args.args[0] + assert payload["actual_write_sources"] == ["cloud_sql"] + assert payload["failure_category"] == category + assert payload["destination_association_id"] is None + assert "secret" not in repr(payload) + assert "credential" not in repr(payload) + + +def test_processing_marker_commit_failure_logs_error_after_supabase_commit( + orm_session_factory, +) -> None: + event_service = UserPolicyService(orm_session_factory) + creation = event_service.create_or_get_user_policy( + _saved_values(), + record_mirror_event=True, + ) + mirror = MagicMock() + mirror.mirror_legacy_user_policy.return_value = LegacyUserPolicyPersistenceResult( + association_id=ASSOCIATION_ID, + policy_id=POLICY_ID, + association_created=True, + association_updated=False, + mapping_created=True, + ) + source_commit_error = OperationalError( + "processed_at update secret", + {"caller": "caller data secret"}, + Exception("database credential"), + ) + + def fail_source_commit(_session) -> None: + raise source_commit_error + + sqlalchemy_event.listen( + orm_session_factory.class_, + "before_commit", + fail_source_commit, + ) + try: + with ( + patch("policyengine_api.services.user_policy_mirroring.logger") as logger, + pytest.raises(UserPolicyMirrorUnavailableError), + ): + mirror_pending_user_policy_events_after_commit( + "us", + creation.user_policy.id, + through_revision=creation.mirror_revision, + event_service=event_service, + reform_snapshot_loader=lambda _country_id, _policy_id: _reform(), + mirror_factory=lambda: mirror, + ) + finally: + sqlalchemy_event.remove( + orm_session_factory.class_, + "before_commit", + fail_source_commit, + ) + + logger.log_struct.assert_called_once() + payload = logger.log_struct.call_args.args[0] + assert payload["outcome"] == "error" + assert payload["failure_category"] == "database" + assert payload["actual_write_sources"] == ["cloud_sql", "supabase"] + assert payload["destination_association_id"] == str(ASSOCIATION_ID) + assert payload["destination_policy_id"] == str(POLICY_ID) + assert "secret" not in repr(payload) + assert "credential" not in repr(payload) + with orm_session_factory() as session: + retained_event = session.scalar(select(UserPolicyMirrorEvent)) + assert retained_event.processed_at is None + + +def test_event_preparation_failure_logs_error_and_retains_the_event( + orm_session_factory, +) -> None: + event_service = UserPolicyService(orm_session_factory) + creation = event_service.create_or_get_user_policy( + _saved_values(), + record_mirror_event=True, + ) + mirror_factory = MagicMock() + + with ( + patch("policyengine_api.services.user_policy_mirroring.logger") as logger, + pytest.raises(UserPolicyMirrorUnavailableError), + ): + mirror_pending_user_policy_events_after_commit( + "us", + creation.user_policy.id, + through_revision=creation.mirror_revision, + event_service=event_service, + reform_snapshot_loader=lambda _country_id, _policy_id: None, + mirror_factory=mirror_factory, + ) + + mirror_factory.assert_not_called() + logger.log_struct.assert_called_once() + payload = logger.log_struct.call_args.args[0] + assert payload["outcome"] == "error" + assert payload["actual_write_sources"] == ["cloud_sql"] + assert payload["source_revision"] == creation.mirror_revision + with orm_session_factory() as session: + retained_event = session.scalar(select(UserPolicyMirrorEvent)) + assert retained_event.processed_at is None diff --git a/tests/unit/services/test_user_policy_service.py b/tests/unit/services/test_user_policy_service.py index 44a96f363..83b6380d9 100644 --- a/tests/unit/services/test_user_policy_service.py +++ b/tests/unit/services/test_user_policy_service.py @@ -1,16 +1,35 @@ import inspect from pathlib import Path +from unittest.mock import MagicMock, patch +from uuid import UUID -from policyengine_api.data.v1_models import UserPolicy +import pytest +from sqlalchemy import event as sqlalchemy_event +from sqlalchemy import func, select +from sqlalchemy.exc import ( + IntegrityError, + OperationalError, + SQLAlchemyError, + TimeoutError as SQLAlchemyTimeoutError, +) + +from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent +from policyengine_api.data.v2.user_policies.legacy import ( + LegacyUserPolicyPersistenceResult, +) from policyengine_api.services.user_policy_service import ( UserPolicyCreateResult, + UserPolicyMirrorEventIntegrityError, + UserPolicyPersistenceError, UserPolicyService, + UserPolicyUpdateResult, ) ROUTE_PATH = ( Path(__file__).parents[3] / "policyengine_api" / "routes" / "policy_routes.py" ) +APPLICATION_ROOT = Path(__file__).parents[3] / "policyengine_api" def _values(**overrides): @@ -55,6 +74,79 @@ def test_policy_routes_do_not_manage_sessions_or_queries(): assert "select(" not in source +@pytest.mark.parametrize( + ("error", "expected_category", "expected_retryable"), + ( + ( + SQLAlchemyTimeoutError("credential=timeout-secret"), + "timeout", + True, + ), + ( + OperationalError( + "UPDATE private_table SET token=:token", + {"token": "bound-parameter-secret"}, + RuntimeError("driver-secret"), + ), + "unavailable", + True, + ), + ( + IntegrityError( + "INSERT caller-private-data", + {"user_id": "caller-private-data"}, + RuntimeError("integrity-secret"), + ), + "integrity", + False, + ), + (SQLAlchemyError("database-secret"), "database", False), + (RuntimeError("unexpected-secret"), "unexpected", False), + ), +) +def test_saved_policy_service_translates_failures_to_safe_domain_errors( + error, + expected_category, + expected_retryable, +) -> None: + session_factory = MagicMock() + session_factory.begin.side_effect = error + service = UserPolicyService(session_factory) + + with pytest.raises(UserPolicyPersistenceError) as raised: + service.create_or_get_user_policy(_values()) + + assert raised.value.category == expected_category + assert raised.value.retryable is expected_retryable + assert str(raised.value) == "Saved-policy persistence failed" + + +def test_saved_policy_persistence_failure_category_is_allowlisted() -> None: + with pytest.raises(ValueError): + UserPolicyPersistenceError("raw-exception-text") # type: ignore[arg-type] + + +def test_saved_policy_event_processing_has_only_request_path_callers(): + sources = { + path.relative_to(APPLICATION_ROOT).as_posix(): path.read_text(encoding="utf-8") + for path in APPLICATION_ROOT.rglob("*.py") + } + + assert { + path + for path, source in sources.items() + if ".process_pending_mirror_events(" in source + } == {"services/user_policy_mirroring.py"} + assert { + path + for path, source in sources.items() + if "mirror_pending_user_policy_events_after_commit(" in source + } == { + "routes/policy_routes.py", + "services/user_policy_mirroring.py", + } + + def test_create_reuse_list_and_update_user_policy(orm_session_factory): service = UserPolicyService(orm_session_factory) @@ -73,10 +165,16 @@ def test_create_reuse_list_and_update_user_policy(orm_session_factory): assert created.created is True assert reused.created is False assert reused.user_policy.id == created.user_policy.id + assert reused.snapshot == created.snapshot assert len(listed) == 1 assert isinstance(listed[0], UserPolicy) - assert updated.reform_label == "Updated" - assert updated.updated_date == 3 + assert isinstance(updated, UserPolicyUpdateResult) + assert updated.user_policy.reform_label == "Updated" + assert updated.user_policy.updated_date == 3 + assert updated.snapshot.reform_label == "Updated" + assert updated.snapshot.updated_date == 3 + assert updated.snapshot.legacy_user_policy_id == created.user_policy.id + assert updated.changed_fields == frozenset({"reform_label", "updated_date"}) def test_update_user_policy_requires_matching_country(orm_session_factory): @@ -93,3 +191,252 @@ def test_update_user_policy_requires_matching_country(orm_session_factory): ) stored = service.list_user_policies("uk", "auth0|one")[0] assert stored.reform_label == "Reform" + + +def test_dual_write_mutations_store_ordered_complete_events_atomically( + orm_session_factory, +): + service = UserPolicyService(orm_session_factory) + + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + updated = service.update_user_policy( + "us", + created.user_policy.id, + {"reform_label": None, "updated_date": 3}, + record_mirror_event=True, + ) + + assert created.mirror_revision == 1 + assert updated is not None + assert updated.mirror_revision == 2 + with orm_session_factory() as session: + stored = session.get(UserPolicy, created.user_policy.id) + events = session.scalars( + select(UserPolicyMirrorEvent).order_by( + UserPolicyMirrorEvent.source_revision + ) + ).all() + + assert stored.mirror_revision == 2 + assert [event.source_revision for event in events] == [1, 2] + assert [event.event_type for event in events] == ["create", "update"] + assert events[0].payload_json["changed_fields"] == [] + assert events[1].payload_json["changed_fields"] == [ + "reform_label", + "updated_date", + ] + assert events[1].payload_json["snapshot"]["reform_label"] is None + assert all(len(event.source_fingerprint_sha256) == 64 for event in events) + assert all(event.processed_at is None for event in events) + + +def test_event_failure_rolls_back_the_source_mutation(orm_session_factory): + service = UserPolicyService(orm_session_factory) + + with ( + patch( + "policyengine_api.services.user_policy_service." + "fingerprint_legacy_user_policy", + side_effect=RuntimeError("cannot encode event"), + ), + pytest.raises(UserPolicyPersistenceError) as raised, + ): + service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + + assert raised.value.category == "unexpected" + assert isinstance(raised.value.__cause__, RuntimeError) + with orm_session_factory() as session: + assert session.scalar(select(func.count()).select_from(UserPolicy)) == 0 + assert ( + session.scalar(select(func.count()).select_from(UserPolicyMirrorEvent)) == 0 + ) + + +def test_update_event_failure_rolls_back_the_source_update(orm_session_factory): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy(_values()) + + with ( + patch( + "policyengine_api.services.user_policy_service." + "fingerprint_legacy_user_policy", + side_effect=RuntimeError("cannot encode event"), + ), + pytest.raises(UserPolicyPersistenceError) as raised, + ): + service.update_user_policy( + "us", + created.user_policy.id, + {"reform_label": "Should roll back"}, + record_mirror_event=True, + ) + + assert raised.value.category == "unexpected" + assert isinstance(raised.value.__cause__, RuntimeError) + with orm_session_factory() as session: + stored = session.get(UserPolicy, created.user_policy.id) + assert stored.reform_label == "Reform" + assert stored.mirror_revision == 0 + assert ( + session.scalar(select(func.count()).select_from(UserPolicyMirrorEvent)) == 0 + ) + + +def test_pending_events_process_in_revision_order_and_mark_after_success( + orm_session_factory, +): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + updated = service.update_user_policy( + "us", + created.user_policy.id, + {"reform_label": "Second"}, + record_mirror_event=True, + ) + processed = [] + committed = [] + + def process(event): + processed.append((event.source_revision, event.changed_fields)) + return LegacyUserPolicyPersistenceResult( + association_id=UUID("00000000-0000-0000-0000-000000000060"), + policy_id=UUID("00000000-0000-0000-0000-000000000010"), + association_created=event.source_revision == 1, + association_updated=event.source_revision == 2, + mapping_created=event.source_revision == 1, + ) + + def record_source_commit(_session): + committed.append(("source_commit", len(processed))) + + def record_processed_commit(event, _result): + committed.append(("completion_callback", event.source_revision)) + + sqlalchemy_event.listen( + orm_session_factory.class_, + "after_commit", + record_source_commit, + ) + try: + result = service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=updated.mirror_revision, + processor=process, + after_processed_commit=record_processed_commit, + ) + finally: + sqlalchemy_event.remove( + orm_session_factory.class_, + "after_commit", + record_source_commit, + ) + + assert processed == [ + (1, frozenset()), + (2, frozenset({"reform_label"})), + ] + assert committed == [ + ("source_commit", 1), + ("completion_callback", 1), + ("source_commit", 2), + ("completion_callback", 2), + ] + assert result.association_updated is True + with orm_session_factory() as session: + events = session.scalars(select(UserPolicyMirrorEvent)).all() + assert all(event.processed_at is not None for event in events) + + +def test_failed_processing_retains_the_oldest_pending_event(orm_session_factory): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + + with pytest.raises(RuntimeError, match="Supabase unavailable"): + service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + processor=lambda event: (_ for _ in ()).throw( + RuntimeError("Supabase unavailable") + ), + ) + + with orm_session_factory() as session: + event = session.scalar(select(UserPolicyMirrorEvent)) + assert event.processed_at is None + + +def test_already_processed_request_revision_is_verified_by_idempotent_replay( + orm_session_factory, +): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + calls = [] + + def process(event): + calls.append(event.source_revision) + return LegacyUserPolicyPersistenceResult( + association_id=UUID("00000000-0000-0000-0000-000000000060"), + policy_id=UUID("00000000-0000-0000-0000-000000000010"), + association_created=len(calls) == 1, + association_updated=False, + mapping_created=len(calls) == 1, + ) + + service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + processor=process, + ) + replay = service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + processor=process, + ) + + assert calls == [1, 1] + assert replay.association_created is False + + +def test_corrupt_event_changed_fields_are_rejected(orm_session_factory): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + with orm_session_factory.begin() as session: + event = session.scalar(select(UserPolicyMirrorEvent)) + event.event_type = "update" + event.payload_json = { + **event.payload_json, + "changed_fields": ["not_a_mutable_source_field"], + } + + with pytest.raises( + UserPolicyMirrorEventIntegrityError, + match="changed fields are unsupported", + ): + service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + processor=lambda event: None, + ) diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index 9517db7bc..431107bdd 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -10,13 +10,16 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["version"] == 1 assert payload["metadata"] == { "route_group_count": 9, - "workflow_count": 8, - "request_count": 32, + "workflow_count": 11, + "request_count": 43, "db_entity_count": 6, "sim_flow_count": 3, } assert {workflow["name"] for workflow in payload["workflows"]} == { "policy_save_search", + "policy_resources_v2", + "saved_policy_v1_compatibility", + "user_policy_associations_v2", "household_save_edit_read", "household_calculate", "region_selection", diff --git a/tests/unit/test_migration_flags.py b/tests/unit/test_migration_flags.py index c70ec95ff..09ddc9bd2 100644 --- a/tests/unit/test_migration_flags.py +++ b/tests/unit/test_migration_flags.py @@ -104,6 +104,25 @@ def test_invalid_migration_flag_raises(monkeypatch): get_migration_context("policy") +def test_phase10_v1_policy_sources_allow_only_cloud_sql_reads_and_mirroring( + monkeypatch, +): + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + + assert migration_flags.get_v1_policy_write_source() == "dual_write" + assert migration_flags.get_v1_policy_read_source() == "cloud_sql" + + monkeypatch.setenv("DB_WRITE_POLICY", "supabase") + with pytest.raises(ValueError, match="DB_WRITE_POLICY"): + migration_flags.get_v1_policy_write_source() + + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + monkeypatch.setenv("DB_READ_POLICY", "read_compare") + with pytest.raises(ValueError, match="DB_READ_POLICY"): + migration_flags.get_v1_policy_read_source() + + @pytest.mark.parametrize( "explicit_sources", [ @@ -134,6 +153,13 @@ def test_explicit_migration_context_rejects_invalid_database_sources( ("/v2/variables", "metadata"), ("/v2/parameters/children", "metadata"), ("/v2/regions/state%2Fca", "metadata"), + ("/v2/policies", "policy"), + ("/v2/policies/00000000-0000-0000-0000-000000000001", "policy"), + ("/v2/user-policies", "policy"), + ( + "/v2/user-policies/00000000-0000-0000-0000-000000000001", + "policy", + ), ("/us/policy/1", "policy"), ("/us/policies", "policy"), ("/us/household/1", "household"), diff --git a/tests/unit/test_query_parameters.py b/tests/unit/test_query_parameters.py new file mode 100644 index 000000000..6313286b7 --- /dev/null +++ b/tests/unit/test_query_parameters.py @@ -0,0 +1,221 @@ +"""Shared query-schema and framework-adapter tests.""" + +from __future__ import annotations + +from typing import Annotated +from uuid import UUID, uuid4 + +from fastapi import Depends, FastAPI +from fastapi.testclient import TestClient +from pydantic import Field, ValidationError +import pytest +from werkzeug.datastructures import MultiDict + +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + CountryQuery, + DuplicateScalarQueryParameterError, + PolicyCollectionQuery, + PolicyCreateQuery, + ResourceId, + UserPolicyCollectionQuery, + parse_multidict_query, + parse_query_items, +) + + +class ExplicitListQuery(CountryQuery): + policy_ids: Annotated[ + list[ResourceId], + Field(default_factory=list, max_length=10), + ] + + +def test_canonical_required_defaults_and_normalization() -> None: + parsed = parse_query_items(PolicyCollectionQuery, [("country_id", "US")]) + + assert parsed.country_id == "us" + assert parsed.offset == 0 + assert parsed.limit == 100 + assert parsed.tax_benefit_model_id is None + + with pytest.raises(ValidationError) as missing: + parse_query_items(PolicyCollectionQuery, []) + assert missing.value.errors()[0]["loc"] == ("country_id",) + + +@pytest.mark.parametrize( + ("items", "field"), + [ + ([("country_id", "ca")], "country_id"), + ([("country_id", "us"), ("offset", "-1")], "offset"), + ([("country_id", "us"), ("limit", "0")], "limit"), + ([("country_id", "us"), ("limit", "501")], "limit"), + ( + [("country_id", "us"), ("tax_benefit_model_id", "not-a-uuid")], + "tax_benefit_model_id", + ), + ([("country_id", "us"), ("unexpected", "value")], "unexpected"), + ], +) +def test_canonical_malformed_out_of_range_and_unknown_values( + items: list[tuple[str, str]], + field: str, +) -> None: + with pytest.raises(ValidationError) as raised: + parse_query_items(PolicyCollectionQuery, items) + assert raised.value.errors()[0]["loc"] == (field,) + + +@pytest.mark.parametrize("version", ["", " 5.2.0", "v5.2.0", "0.0.0"]) +def test_policyengine_version_must_be_canonical(version: str) -> None: + with pytest.raises(ValidationError): + parse_query_items( + PolicyCreateQuery, + [("country_id", "uk"), ("policyengine_version", version)], + ) + + assert ( + parse_query_items( + PolicyCreateQuery, + [("country_id", "uk"), ("policyengine_version", "5.2.0")], + ).policyengine_version + == "5.2.0" + ) + + +def test_duplicate_scalar_is_rejected_and_explicit_list_is_preserved() -> None: + with pytest.raises( + DuplicateScalarQueryParameterError, + match="country_id.*must not be repeated", + ): + parse_query_items( + PolicyCollectionQuery, + [("country_id", "us"), ("country_id", "uk")], + ) + + first_id = uuid4() + second_id = uuid4() + parsed = parse_query_items( + ExplicitListQuery, + [ + ("country_id", "us"), + ("policy_ids", str(first_id)), + ("policy_ids", str(second_id)), + ], + ) + assert parsed.policy_ids == [first_id, second_id] + + +def test_multidict_adapter_preserves_the_canonical_contract() -> None: + policy_id = uuid4() + parsed = parse_multidict_query( + UserPolicyCollectionQuery, + MultiDict( + [ + ("country_id", "UK"), + ("user_id", "auth0|example"), + ("policy_id", str(policy_id)), + ("offset", "2"), + ] + ), + ) + + assert parsed.country_id == "uk" + assert parsed.user_id == "auth0|example" + assert parsed.policy_id == policy_id + assert parsed.offset == 2 + + +def _test_client() -> TestClient: + app = FastAPI() + dependency = query_dependency(ExplicitListQuery) + + @app.get("/resources") + def resources( + query: ExplicitListQuery = Depends(dependency), + ) -> dict[str, object]: + return query.model_dump(mode="json") + + return TestClient(app) + + +def test_fastapi_dependency_rejects_unknown_and_duplicate_scalar_parameters() -> None: + client = _test_client() + + assert client.get("/resources?country_id=us&unknown=value").status_code == 422 + assert client.get("/resources?country_id=us&country_id=uk").status_code == 422 + + +def test_fastapi_dependency_accepts_repeated_explicit_list_parameters() -> None: + first_id = uuid4() + second_id = uuid4() + + response = _test_client().get( + f"/resources?country_id=US&policy_ids={first_id}&policy_ids={second_id}" + ) + + assert response.status_code == 200 + assert response.json() == { + "country_id": "us", + "policy_ids": [str(first_id), str(second_id)], + } + + +def test_openapi_matches_composed_runtime_query_contract() -> None: + operation = _test_client().get("/openapi.json").json()["paths"]["/resources"]["get"] + parameters = {item["name"]: item for item in operation["parameters"]} + + assert set(parameters) == {"country_id", "policy_ids"} + assert parameters["country_id"]["required"] is True + assert parameters["country_id"]["schema"]["enum"] == ["us", "uk"] + assert parameters["policy_ids"]["required"] is False + assert parameters["policy_ids"]["schema"]["type"] == "array" + assert parameters["policy_ids"]["schema"]["maxItems"] == 10 + assert parameters["policy_ids"]["schema"]["items"]["format"] == "uuid" + + +def test_policy_collection_openapi_defaults_and_bounds() -> None: + app = FastAPI() + dependency = query_dependency(PolicyCollectionQuery) + + @app.get("/policies") + def policies( + query: PolicyCollectionQuery = Depends(dependency), + ) -> dict[str, object]: + return query.model_dump(mode="json") + + operation = TestClient(app).get("/openapi.json").json()["paths"]["/policies"]["get"] + parameters = {item["name"]: item for item in operation["parameters"]} + + assert parameters["offset"]["schema"] == { + "type": "integer", + "minimum": 0, + "description": "Zero-based result offset", + "default": 0, + "title": "Offset", + } + assert parameters["limit"]["schema"] == { + "type": "integer", + "maximum": 500, + "minimum": 1, + "description": "Maximum number of returned resources", + "default": 100, + "title": "Limit", + } + assert parameters["tax_benefit_model_id"]["required"] is False + uuid_schema = parameters["tax_benefit_model_id"]["schema"]["anyOf"][0] + assert uuid_schema == { + "type": "string", + "format": "uuid", + "description": "Exact resource UUID", + } + + +def test_uuid_filter_is_materialized_as_uuid() -> None: + model_id = uuid4() + parsed = parse_query_items( + PolicyCollectionQuery, + [("country_id", "us"), ("tax_benefit_model_id", str(model_id))], + ) + assert isinstance(parsed.tax_benefit_model_id, UUID) diff --git a/tests/unit/test_readiness.py b/tests/unit/test_readiness.py index 1f5f706c6..3cc29168c 100644 --- a/tests/unit/test_readiness.py +++ b/tests/unit/test_readiness.py @@ -1,4 +1,5 @@ import pytest +from unittest.mock import patch from policyengine_api import readiness @@ -19,3 +20,48 @@ def test_mark_not_ready_then_ready(): assert readiness.is_ready() is False readiness.mark_ready() assert readiness.is_ready() is True + + +def test_default_cloud_sql_policy_mode_does_not_require_v2_settings( + monkeypatch, +): + monkeypatch.delenv("DB_WRITE_POLICY", raising=False) + monkeypatch.delenv("DB_READ_POLICY", raising=False) + monkeypatch.delenv("ROUTE_IMPL_POLICY", raising=False) + monkeypatch.delenv("V2_RUNTIME_DATABASE_URL", raising=False) + monkeypatch.delenv("V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", raising=False) + + readiness.mark_ready() + + assert readiness.is_ready() is True + + +def test_dual_write_or_native_policy_routes_require_v2_runtime_settings( + monkeypatch, +): + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + monkeypatch.delenv("V2_RUNTIME_DATABASE_URL", raising=False) + monkeypatch.delenv("V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", raising=False) + readiness.mark_ready() + + assert readiness.is_ready() is False + + +def test_selected_v2_policy_modes_validate_runtime_settings(monkeypatch): + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + monkeypatch.setenv("ROUTE_IMPL_POLICY", "flask_fallback") + readiness.mark_ready() + + with patch( + "policyengine_api.data.v2.settings.load_v2_runtime_database_settings", + return_value=object(), + ) as load_settings: + assert readiness.is_ready() is True + + load_settings.assert_called_once_with() + + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + monkeypatch.setenv("ROUTE_IMPL_POLICY", "fastapi_native") + + assert readiness.is_ready() is False diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 815acf476..8ed46ac87 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -217,11 +217,13 @@ def test_v2_files_are_mechanically_separate_from_v1() -> None: assert all("migrations/v1" not in str(path) for path in v2_files) -def test_v2_revision_chain_has_baseline_and_generated_stage_9_revision() -> None: +def test_v2_revision_chain_has_generated_baseline_stage_9_and_phase_10() -> None: config = Config(str(REPO / "alembic-v2.ini")) script = ScriptDirectory.from_config(config) - assert script.get_heads() == ["68b4a5ae5dc5"] + assert script.get_heads() == ["c21c4a807a49"] assert [revision.revision for revision in script.walk_revisions()] == [ + "c21c4a807a49", + "711ec2f0a5a5", "68b4a5ae5dc5", "f5ef4347cb2a", ] @@ -248,8 +250,8 @@ def test_v2_revision_chain_has_baseline_and_generated_stage_9_revision() -> None assert "fk_regions_default_dataset_model_datasets" in baseline assert "uq_datasets_model_name" in baseline assert "ck_datasets_output_storage_path" in baseline - assert baseline.count("op.create_table(") == len(V2_TABLE_NAMES) - assert baseline.count("op.drop_table(") == len(V2_TABLE_NAMES) + assert baseline.count("op.create_table(") == len(V2_TABLE_NAMES) - 2 + assert baseline.count("op.drop_table(") == len(V2_TABLE_NAMES) - 2 corrected_enum_names = set( re.findall( @@ -288,6 +290,59 @@ def test_v2_revision_chain_has_baseline_and_generated_stage_9_revision() -> None assert "op.bulk_insert(" not in stage_9_revision +def test_phase_10_revision_has_only_documented_generation_corrections() -> None: + revision = ( + REPO / "migrations/v2/versions/711ec2f0a5a5_migrate_v2_policies.py" + ).read_text(encoding="utf-8") + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert 'down_revision: Union[str, None] = "68b4a5ae5dc5"' in revision + assert revision.count("Post-generation correction:") == 4 + assert 'postgresql_using="user_id::text"' in revision + assert 'postgresql_using="user_id::uuid"' in revision + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision + + policy_key = revision.index('"uq_policies_id_country"') + policy_mapping = revision.index( + 'op.create_table(\n "legacy_policy_mappings"' + ) + association_key = revision.index('"uq_user_policies_id_country"') + association_mapping = revision.index( + 'op.create_table(\n "legacy_user_policy_mappings"' + ) + drop_association_mapping = revision.index( + 'op.drop_table("legacy_user_policy_mappings")' + ) + drop_association_key = revision.index( + 'op.drop_constraint("uq_user_policies_id_country"' + ) + + assert policy_key < policy_mapping + assert association_key < association_mapping + assert drop_association_mapping < drop_association_key + + +def test_saved_policy_revision_tracking_was_generated_after_phase_10() -> None: + revision = ( + REPO + / "migrations/v2/versions/c21c4a807a49_track_saved_policy_mirror_revisions.py" + ).read_text(encoding="utf-8") + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert 'down_revision: Union[str, None] = "711ec2f0a5a5"' in revision + assert "last_applied_source_revision" in revision + assert "ck_legacy_user_policy_mappings_source_revision" in revision + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision + + def test_alembic_rejects_unknown_missing_and_divergent_history(tmp_path: Path) -> None: original = REPO / "migrations/v2" missing = tmp_path / "missing" diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py index e9a153e38..bbc8179de 100644 --- a/tests/unit/v2/test_database.py +++ b/tests/unit/v2/test_database.py @@ -50,6 +50,34 @@ def test_engine_construction_is_lazy_and_reused_without_connecting() -> None: assert first.pool.checkedout() == 0 +def test_engine_uses_one_timeout_for_pool_connection_and_statements( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = load_v2_runtime_database_settings(_environment()) + captured: dict[str, object] = {} + expected_engine = object() + + def capture_create_engine(url: object, **options: object) -> object: + captured["url"] = url + captured.update(options) + return expected_engine + + monkeypatch.setattr(database, "create_engine", capture_create_engine) + + engine = database.build_v2_engine(settings) + + assert engine is expected_engine + assert captured["url"] is settings.connection.url + assert database.DATABASE_TIMEOUT_SECONDS == 5 + assert captured["pool_timeout"] == database.DATABASE_TIMEOUT_SECONDS + assert captured["connect_args"] == { + "connect_timeout": database.DATABASE_TIMEOUT_SECONDS, + "options": ( + f"-c statement_timeout={database.DATABASE_TIMEOUT_SECONDS * 1_000}" + ), + } + + def test_session_factory_builds_sqlmodel_sessions() -> None: settings = load_v2_runtime_database_settings(_environment()) factory = database.get_v2_session_factory(settings) diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index d64af2741..6d27d6ecd 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -50,11 +50,16 @@ def test_importing_v2_modules_opens_no_network_and_creates_no_files( script = """ import pathlib import socket +import sqlalchemy def reject_connect(*args, **kwargs): raise AssertionError("module import attempted a network connection") +def reject_ddl(*args, **kwargs): + raise AssertionError("module import attempted implicit DDL") + socket.socket.connect = reject_connect +sqlalchemy.MetaData.create_all = reject_ddl before = set(pathlib.Path.cwd().iterdir()) import policyengine_api.data.v2.settings import policyengine_api.data.v2.database @@ -62,8 +67,9 @@ def reject_connect(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 32 +assert len(V2_METADATA.tables) == 34 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules +assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules """ result = subprocess.run( diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 04c960969..457c22534 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -515,7 +515,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: assert response.status_code == 200 schema = response.json() - expected_paths = { + metadata_paths = { "/v2/datasets", "/v2/datasets/{dataset_id}", "/v2/economy-options", @@ -535,9 +535,15 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/variables", "/v2/variables/{variable_id}", } - assert set(schema["paths"]) == expected_paths + native_paths = { + "/v2/policies", + "/v2/policies/{policy_id}", + "/v2/user-policies", + "/v2/user-policies/{association_id}", + } + assert set(schema["paths"]) == metadata_paths | native_paths - for path in expected_paths: + for path in metadata_paths: operation = schema["paths"][path]["get"] assert set(operation["responses"]) >= { "200", diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index efbaa5c05..c8a7f376e 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -1,23 +1,64 @@ """Canonical SQLModel persistence and bounded SQLAlchemy escape-hatch tests.""" -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path import pytest +import sqlalchemy as sa from sqlalchemy.exc import IntegrityError from sqlmodel import Session, create_engine, select from policyengine_api.data.v2.models import ( + Dynamic, + LegacyPolicyMapping, + LegacyUserPolicyMapping, Parameter, ParameterValue, + Policy, TaxBenefitModel, TaxBenefitModelVersion, User, + UserPolicy, V2_METADATA, ) from policyengine_api.data.v2.models.base import DIRECT_SQLALCHEMY_EXCEPTIONS +def _relational_sqlite_engine(): + engine = create_engine("sqlite://") + + @sa.event.listens_for(engine, "connect") + def enable_foreign_keys(dbapi_connection, _connection_record) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + V2_METADATA.create_all(engine) + return engine + + +def _policy_graph(*, content_hash: str = "a" * 64): + model = TaxBenefitModel(name=f"policy-model-{content_hash[:8]}") + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.example.rate", + tax_benefit_model_version=version, + ) + policy = Policy( + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash=content_hash, + ) + return model, version, parameter, policy + + def test_ordinary_persistence_uses_sqlmodel_session_select_and_exec() -> None: # SQLite exists only as an injected, in-memory unit-test fixture. Runtime # v2 settings reject it and application code never selects it. @@ -121,6 +162,209 @@ def test_canonical_parameter_values_are_unique_by_parameter_and_start_date() -> engine.dispose() +def test_policy_content_hash_is_unique_per_canonicalization_version() -> None: + engine = _relational_sqlite_engine() + model, version, _parameter, policy = _policy_graph() + duplicate = Policy( + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=policy.canonicalization_version, + content_hash=policy.content_hash, + ) + + with Session(engine) as session: + session.add_all([policy, duplicate]) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_policy_parameter_value_owner_period_and_identity_constraints() -> None: + engine = _relational_sqlite_engine() + _model, _version, parameter, policy = _policy_graph(content_hash="b" * 64) + dynamic = Dynamic(name="dynamic") + start = datetime(2026, 1, 1, tzinfo=timezone.utc) + + with Session(engine) as session: + session.add_all([parameter, policy, dynamic]) + session.commit() + + invalid_owner = ParameterValue( + parameter_id=parameter.id, + policy_id=policy.id, + dynamic_id=dynamic.id, + value_json={"rate": 0.1}, + start_date=start, + ) + session.add(invalid_owner) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + invalid_period = ParameterValue( + parameter_id=parameter.id, + policy_id=policy.id, + value_json={"rate": 0.1}, + start_date=start, + end_date=start - timedelta(seconds=1), + ) + session.add(invalid_period) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + session.add_all( + [ + ParameterValue( + parameter_id=parameter.id, + policy_id=policy.id, + value_json={"rate": 0.1}, + start_date=start, + ), + ParameterValue( + parameter_id=parameter.id, + policy_id=policy.id, + value_json={"rate": 0.2}, + start_date=start, + ), + ] + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_user_policy_allows_duplicates_but_requires_policy_country() -> None: + engine = _relational_sqlite_engine() + _model, _version, _parameter, policy = _policy_graph(content_hash="c" * 64) + + with Session(engine) as session: + session.add(policy) + session.commit() + first = UserPolicy( + country_id="us", + user_id="auth0|caller", + policy_id=policy.id, + name="First", + ) + second = UserPolicy( + country_id="us", + user_id="auth0|caller", + policy_id=policy.id, + name="Second", + ) + session.add_all([first, second]) + session.commit() + + assert first.id != second.id + + session.add( + UserPolicy( + country_id="uk", + user_id="auth0|caller", + policy_id=policy.id, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_legacy_policy_mapping_allows_many_sources_for_one_policy() -> None: + engine = _relational_sqlite_engine() + _model, _version, _parameter, policy = _policy_graph(content_hash="d" * 64) + + with Session(engine) as session: + session.add(policy) + session.commit() + session.add_all( + [ + LegacyPolicyMapping( + country_id="us", + legacy_policy_id=101, + policy_id=policy.id, + source_policy_hash="1" * 64, + ), + LegacyPolicyMapping( + country_id="us", + legacy_policy_id=102, + policy_id=policy.id, + source_policy_hash="2" * 64, + ), + ] + ) + session.commit() + + assert len(policy.legacy_mappings) == 2 + + session.add( + LegacyPolicyMapping( + country_id="us", + legacy_policy_id=101, + policy_id=policy.id, + source_policy_hash="3" * 64, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_legacy_user_policy_mapping_destination_is_unique_and_cascades() -> None: + engine = _relational_sqlite_engine() + _model, _version, _parameter, policy = _policy_graph(content_hash="e" * 64) + + with Session(engine) as session: + association = UserPolicy( + country_id="us", + user_id="legacy-user", + policy=policy, + ) + mapping = LegacyUserPolicyMapping( + country_id="us", + legacy_user_policy_id=201, + association=association, + fingerprint_version=1, + fingerprint_sha256="4" * 64, + ) + session.add(mapping) + session.commit() + assert mapping.created_at is not None + assert mapping.updated_at is not None + + session.add( + LegacyUserPolicyMapping( + country_id="us", + legacy_user_policy_id=202, + user_policy_id=association.id, + fingerprint_version=1, + fingerprint_sha256="5" * 64, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + session.delete(association) + session.commit() + stored_mapping = session.exec( + select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.id == mapping.id + ) + ).one_or_none() + assert stored_mapping is None + + stored_policy = session.get(Policy, policy.id) + assert stored_policy is not None + + engine.dispose() + + def test_v2_models_do_not_create_a_parallel_sqlalchemy_orm_layer() -> None: models_directory = ( Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" / "models" diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index dbdab10ee..ff2ba189b 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -11,6 +11,9 @@ Dynamic, Household, HouseholdJob, + LegacyPolicyMapping, + LegacyUserPolicyMapping, + ParameterValue, Policy, Simulation, TaxBenefitModelVersion, @@ -50,6 +53,8 @@ def test_domain_models_are_grouped_into_topic_scoped_modules() -> None: Simulation: "simulations", UserHouseholdAssociation: "associations", UserPolicy: "associations", + LegacyPolicyMapping: "policy_mappings", + LegacyUserPolicyMapping: "policy_mappings", UserReportAssociation: "associations", UserSimulationAssociation: "associations", } @@ -105,7 +110,7 @@ def test_every_declared_relationship_has_a_complete_back_populates_pair() -> Non assert inverse.mapper is mapper -def test_every_user_association_has_relational_integrity() -> None: +def test_user_owned_associations_have_relational_integrity() -> None: configure_mappers() user_mapper = sa.inspect(User) associations = ( @@ -114,7 +119,6 @@ def test_every_user_association_has_relational_integrity() -> None: "user_household_associations", "household_associations", ), - (UserPolicy, "user_policies", "policy_associations"), ( UserSimulationAssociation, "user_simulation_associations", @@ -135,6 +139,124 @@ def test_every_user_association_has_relational_integrity() -> None: assert user_mapper.relationships[user_collection].back_populates == "user" +def test_policy_content_identity_and_catalog_columns_are_explicit() -> None: + policies = V2_METADATA.tables["policies"] + + assert {"name", "description"}.isdisjoint(policies.c.keys()) + assert { + "country_id", + "tax_benefit_model_id", + "tax_benefit_model_version_id", + "canonicalization_version", + "content_hash", + "created_at", + "updated_at", + }.issubset(policies.c.keys()) + assert policies.c.country_id.type.length == 2 + assert policies.c.content_hash.type.length == 64 + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in policies.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("id", "country_id") in unique_columns + assert ("canonicalization_version", "content_hash") in unique_columns + assert { + "ix_policies_country_model", + "ix_policies_country_model_version", + } <= {index.name for index in policies.indexes} + + +def test_policy_parameter_values_use_jsonb_and_enforce_period_identity() -> None: + values = V2_METADATA.tables["parameter_values"] + + postgres_type = values.c.value_json.type.dialect_impl(postgresql.dialect()) + assert isinstance(postgres_type, postgresql.JSONB) + assert { + "ck_parameter_values_single_owner", + "ck_parameter_values_effective_period", + } <= {constraint.name for constraint in values.constraints} + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in values.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("policy_id", "parameter_id", "start_date") in unique_columns + assert sa.inspect(ParameterValue).relationships["policy"].back_populates == ( + "parameter_values" + ) + + +def test_user_policy_is_an_independent_country_scoped_association() -> None: + associations = V2_METADATA.tables["user_policies"] + + assert associations.c.user_id.type.length == 255 + assert list(associations.c.user_id.foreign_keys) == [] + assert {"country", "label"}.isdisjoint(associations.c.keys()) + assert {"country_id", "name", "description"}.issubset(associations.c.keys()) + assert associations.c.name.nullable + assert associations.c.description.nullable + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in associations.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("user_id", "policy_id") not in unique_columns + assert ("id", "country_id") in unique_columns + policy_country = next( + constraint + for constraint in associations.foreign_key_constraints + if constraint.name == "fk_user_policies_policy_country" + ) + assert [column.name for column in policy_country.columns] == [ + "policy_id", + "country_id", + ] + assert [element.target_fullname for element in policy_country.elements] == [ + "policies.id", + "policies.country_id", + ] + assert policy_country.ondelete == "RESTRICT" + + +def test_legacy_policy_mapping_is_many_to_one_by_destination() -> None: + mappings = V2_METADATA.tables["legacy_policy_mappings"] + + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in mappings.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("country_id", "legacy_policy_id") in unique_columns + assert ("policy_id",) not in unique_columns + assert mappings.c.source_policy_hash.type.length == 255 + policy_country = next(iter(mappings.foreign_key_constraints)) + assert policy_country.name == "fk_legacy_policy_mappings_policy_country" + assert policy_country.ondelete == "RESTRICT" + + +def test_legacy_user_policy_mapping_is_one_to_one_by_destination() -> None: + mappings = V2_METADATA.tables["legacy_user_policy_mappings"] + + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in mappings.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("country_id", "legacy_user_policy_id") in unique_columns + assert ("user_policy_id",) in unique_columns + assert mappings.c.fingerprint_sha256.type.length == 64 + assert mappings.c.last_applied_source_revision.server_default.arg == "0" + assert "ck_legacy_user_policy_mappings_source_revision" in { + constraint.name for constraint in mappings.constraints + } + association_country = next(iter(mappings.foreign_key_constraints)) + assert ( + association_country.name == "fk_legacy_user_policy_mappings_association_country" + ) + assert association_country.ondelete == "CASCADE" + + def test_all_datetime_columns_are_timezone_aware() -> None: datetime_columns = [ column diff --git a/tests/unit/v2/test_policy_canonicalization.py b/tests/unit/v2/test_policy_canonicalization.py new file mode 100644 index 000000000..bf20eee9a --- /dev/null +++ b/tests/unit/v2/test_policy_canonicalization.py @@ -0,0 +1,180 @@ +"""Deterministic canonical policy-content tests.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import hashlib +from uuid import UUID, uuid4 + +from policyengine_api.data.v2.policies.canonicalization import ( + POLICY_CANONICALIZATION_VERSION, + canonical_policy_document, + canonicalize_policy, +) +from policyengine_api.data.v2.policies.schemas import ResolvedPolicyCreateCommand + + +MODEL_ID = UUID("00000000-0000-0000-0000-000000000010") +MODEL_VERSION_ID = UUID("00000000-0000-0000-0000-000000000020") +FIRST_PARAMETER_ID = UUID("00000000-0000-0000-0000-000000000030") +SECOND_PARAMETER_ID = UUID("00000000-0000-0000-0000-000000000040") + + +def _command(*, values=None, **changes) -> ResolvedPolicyCreateCommand: + fields = { + "country_id": "us", + "tax_benefit_model_id": MODEL_ID, + "tax_benefit_model_version_id": MODEL_VERSION_ID, + "policyengine_version": "5.2.0", + "parameter_values": values + if values is not None + else [ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"enabled": True, "rate": 1}, + "start_date": "2026-01-01T00:00:00Z", + } + ], + } + fields.update(changes) + return ResolvedPolicyCreateCommand.model_validate(fields) + + +def test_document_has_versioned_deterministic_content_only() -> None: + document = canonical_policy_document(_command()) + + assert document == ( + b'{"canonicalization_version":1,"country_id":"us",' + b'"parameter_values":[{"end_date":null,' + b'"parameter_id":"00000000-0000-0000-0000-000000000030",' + b'"start_date":"2026-01-01T00:00:00.000000Z",' + b'"value":{"enabled":true,"rate":1}}],' + b'"tax_benefit_model_id":"00000000-0000-0000-0000-000000000010",' + b'"tax_benefit_model_version_id":' + b'"00000000-0000-0000-0000-000000000020"}' + ) + assert b"policyengine_version" not in document + assert b"created_at" not in document + assert b"name" not in document + + +def test_request_and_object_member_order_do_not_change_identity() -> None: + first = { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"z": [3, 2, 1], "a": {"right": 2, "left": 1}}, + "start_date": "2026-01-01T00:00:00Z", + } + second = { + "parameter_id": SECOND_PARAMETER_ID, + "value": "second", + "start_date": "2027-01-01T00:00:00Z", + } + reordered_first = { + **first, + "value": {"a": {"left": 1, "right": 2}, "z": [3, 2, 1]}, + } + + assert canonicalize_policy(_command(values=[first, second])) == canonicalize_policy( + _command(values=[second, reordered_first]) + ) + + +def test_equivalent_json_numbers_and_utc_instants_have_one_encoding() -> None: + integer = _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"positive": 1, "zero": 0}, + "start_date": "2026-01-01T00:00:00Z", + } + ] + ) + floating = _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"zero": -0.0, "positive": 1.0}, + "start_date": "2026-01-01T03:00:00+03:00", + } + ] + ) + + assert canonicalize_policy(integer) == canonicalize_policy(floating) + + +def test_material_content_changes_produce_distinct_documents() -> None: + original = canonical_policy_document(_command()) + alternatives = [ + _command(country_id="uk"), + _command(tax_benefit_model_id=uuid4()), + _command(tax_benefit_model_version_id=uuid4()), + _command( + values=[ + { + "parameter_id": SECOND_PARAMETER_ID, + "value": {"enabled": True, "rate": 1}, + "start_date": "2026-01-01T00:00:00Z", + } + ] + ), + _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"enabled": True, "rate": 2}, + "start_date": "2026-01-01T00:00:00Z", + } + ] + ), + _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"enabled": True, "rate": 1}, + "start_date": "2026-01-02T00:00:00Z", + } + ] + ), + _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"enabled": True, "rate": 1}, + "start_date": "2026-01-01T00:00:00Z", + "end_date": "2026-12-31T00:00:00Z", + } + ] + ), + ] + + assert all(canonical_policy_document(item) != original for item in alternatives) + + +def test_digest_is_sha256_of_exact_canonical_bytes() -> None: + content = canonicalize_policy(_command()) + + assert content.version == POLICY_CANONICALIZATION_VERSION + assert content.content_hash == hashlib.sha256(content.document).hexdigest() + assert len(content.content_hash) == 64 + + +def test_datetime_objects_are_rendered_at_fixed_microsecond_precision() -> None: + command = _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": 1, + "start_date": datetime( + 2026, + 1, + 1, + 0, + 0, + 0, + 42, + tzinfo=timezone.utc, + ), + } + ] + ) + assert b"2026-01-01T00:00:00.000042Z" in canonical_policy_document(command) diff --git a/tests/unit/v2/test_policy_catalog.py b/tests/unit/v2/test_policy_catalog.py new file mode 100644 index 000000000..5b82ae1fd --- /dev/null +++ b/tests/unit/v2/test_policy_catalog.py @@ -0,0 +1,152 @@ +"""Exact catalog-binding tests for immutable policies.""" + +from __future__ import annotations + +from uuid import uuid4 + +from sqlmodel import Session, create_engine +import pytest + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.data.v2.models import ( + Parameter, + TaxBenefitModel, + TaxBenefitModelVersion, + V2_METADATA, +) +from policyengine_api.data.v2.policies.catalog import ( + PolicyCatalogValidationError, + resolve_policy_catalog, +) +from policyengine_api.data.v2.policies.schemas import PolicyCreateCommand + + +def _catalog_session(): + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + session = Session(engine) + us_model = TaxBenefitModel(name="policyengine-us") + current = TaxBenefitModelVersion( + model=us_model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + previous = TaxBenefitModelVersion( + model=us_model, + version="5.1.0", + current_law_id=2, + metadata_time_periods=[2025], + ) + current_parameter = Parameter( + name="gov.example.rate", + tax_benefit_model_version=current, + ) + previous_parameter = Parameter( + name="gov.example.old_rate", + tax_benefit_model_version=previous, + ) + session.add_all([current_parameter, previous_parameter]) + session.commit() + return engine, session, us_model, current, current_parameter, previous_parameter + + +def _command(model_id, parameter_id=None, *, country_id="us"): + parameter_values = [] + if parameter_id is not None: + parameter_values.append( + { + "parameter_id": parameter_id, + "value": 0.2, + "start_date": "2026-01-01T00:00:00Z", + } + ) + return PolicyCreateCommand( + country_id=country_id, + tax_benefit_model_id=model_id, + parameter_values=parameter_values, + ) + + +def test_resolver_binds_model_version_and_all_parameter_ids() -> None: + engine, session, model, version, parameter, _previous = _catalog_session() + try: + resolved = resolve_policy_catalog( + session, + _command(model.id, parameter.id), + policyengine_version="5.2.0", + running_policyengine_version="different-running-version", + ) + + assert resolved.country_id == "us" + assert resolved.tax_benefit_model_id == model.id + assert resolved.tax_benefit_model_version_id == version.id + assert resolved.policyengine_version == "5.2.0" + assert resolved.parameter_values[0].parameter_id == parameter.id + finally: + session.close() + engine.dispose() + + +def test_omitted_version_selects_the_running_catalog() -> None: + engine, session, model, version, _parameter, _previous = _catalog_session() + try: + resolved = resolve_policy_catalog( + session, + _command(model.id), + running_policyengine_version="5.2.0", + ) + assert resolved.tax_benefit_model_version_id == version.id + finally: + session.close() + engine.dispose() + + +def test_wrong_stable_model_is_rejected() -> None: + engine, session, _model, _version, parameter, _previous = _catalog_session() + try: + with pytest.raises(PolicyCatalogValidationError, match="selected country"): + resolve_policy_catalog( + session, + _command(uuid4(), parameter.id), + policyengine_version="5.2.0", + ) + finally: + session.close() + engine.dispose() + + +def test_parameter_from_another_model_version_is_rejected() -> None: + engine, session, model, _version, _parameter, previous = _catalog_session() + try: + with pytest.raises(PolicyCatalogValidationError, match="every parameter_id"): + resolve_policy_catalog( + session, + _command(model.id, previous.id), + policyengine_version="5.2.0", + ) + finally: + session.close() + engine.dispose() + + +def test_absent_or_unsupported_catalog_never_falls_back() -> None: + engine, session, model, _version, _parameter, _previous = _catalog_session() + try: + with pytest.raises(MetadataCatalogVersionNotFoundError): + resolve_policy_catalog( + session, + _command(model.id), + policyengine_version="4.0.0", + ) + with pytest.raises(MetadataCatalogVersionNotFoundError): + resolve_policy_catalog( + session, + _command(model.id, country_id="uk"), + policyengine_version="5.2.0", + ) + finally: + session.close() + engine.dispose() diff --git a/tests/unit/v2/test_policy_commands.py b/tests/unit/v2/test_policy_commands.py new file mode 100644 index 000000000..59108bd31 --- /dev/null +++ b/tests/unit/v2/test_policy_commands.py @@ -0,0 +1,146 @@ +"""Validation tests for immutable v2 policy commands.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from uuid import uuid4 + +from pydantic import ValidationError +import pytest + +from policyengine_api.data.v2.policies.schemas import ( + MAXIMUM_POLICY_PARAMETER_VALUES, + NativePolicyCreateCommand, + PolicyCreateCommand, + PolicyParameterValueCommand, + ResolvedPolicyCreateCommand, +) + + +def _value(**changes) -> dict[str, object]: + fields: dict[str, object] = { + "parameter_id": uuid4(), + "value": {"rate": 0.2, "enabled": True}, + "start_date": "2026-01-01T03:00:00+03:00", + "end_date": None, + } + fields.update(changes) + return fields + + +def _command(**changes) -> dict[str, object]: + fields: dict[str, object] = { + "country_id": "US", + "tax_benefit_model_id": uuid4(), + "parameter_values": [_value()], + } + fields.update(changes) + return fields + + +def test_command_normalizes_country_and_effective_dates_to_utc() -> None: + command = PolicyCreateCommand.model_validate(_command()) + + assert command.country_id == "us" + assert command.parameter_values[0].start_date == datetime( + 2026, + 1, + 1, + tzinfo=timezone.utc, + ) + + +def test_native_and_resolved_commands_keep_catalog_selection_explicit() -> None: + native = NativePolicyCreateCommand.model_validate( + {**_command(), "policyengine_version": "5.2.0"} + ) + resolved = ResolvedPolicyCreateCommand.model_validate( + { + **native.model_dump(exclude={"policyengine_version"}), + "policyengine_version": "5.2.0", + "tax_benefit_model_version_id": uuid4(), + } + ) + + assert native.policyengine_version == "5.2.0" + assert resolved.tax_benefit_model_version_id is not None + + +@pytest.mark.parametrize( + "value", + [ + float("nan"), + float("inf"), + float("-inf"), + Decimal("1.2"), + {1: "non-string key"}, + ("tuple",), + object(), + ], +) +def test_non_json_values_are_rejected(value: object) -> None: + with pytest.raises(ValidationError): + PolicyParameterValueCommand.model_validate(_value(value=value)) + + +def test_json_reference_cycles_are_rejected() -> None: + cyclic: list[object] = [] + cyclic.append(cyclic) + + with pytest.raises(ValidationError, match="reference cycles"): + PolicyParameterValueCommand.model_validate(_value(value=cyclic)) + + +@pytest.mark.parametrize( + "changes", + [ + {"start_date": "2026-01-01T00:00:00"}, + { + "start_date": "2026-01-02T00:00:00Z", + "end_date": "2026-01-01T00:00:00Z", + }, + ], +) +def test_invalid_effective_dates_are_rejected(changes: dict[str, object]) -> None: + with pytest.raises(ValidationError): + PolicyParameterValueCommand.model_validate(_value(**changes)) + + +def test_duplicate_parameter_and_normalized_start_date_is_rejected() -> None: + parameter_id = uuid4() + first = _value( + parameter_id=parameter_id, + start_date="2026-01-01T00:00:00Z", + ) + duplicate = _value( + parameter_id=parameter_id, + start_date="2026-01-01T03:00:00+03:00", + ) + + with pytest.raises(ValidationError, match="parameter_id/start_date"): + PolicyCreateCommand.model_validate( + _command(parameter_values=[first, duplicate]) + ) + + +def test_parameter_value_count_is_bounded_but_empty_policy_is_valid() -> None: + assert ( + PolicyCreateCommand.model_validate( + _command(parameter_values=[]) + ).parameter_values + == [] + ) + + repeated = _value() + parameter_values = [ + { + **repeated, + "parameter_id": uuid4(), + "start_date": datetime(2026, 1, 1, tzinfo=timezone.utc) + + timedelta(days=index), + } + for index in range(MAXIMUM_POLICY_PARAMETER_VALUES + 1) + ] + with pytest.raises(ValidationError): + PolicyCreateCommand.model_validate(_command(parameter_values=parameter_values)) diff --git a/tests/unit/v2/test_policy_legacy_translation.py b/tests/unit/v2/test_policy_legacy_translation.py new file mode 100644 index 000000000..f69094a0b --- /dev/null +++ b/tests/unit/v2/test_policy_legacy_translation.py @@ -0,0 +1,201 @@ +"""Legacy policy snapshot and translation tests.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from pydantic import ValidationError +from sqlmodel import Session, create_engine +import pytest + +from policyengine_api.data.v2.models import ( + Parameter, + TaxBenefitModel, + TaxBenefitModelVersion, + V2_METADATA, +) +from policyengine_api.data.v2.policies.legacy import ( + LegacyPolicySnapshot, + LegacyPolicyTranslationError, + parse_legacy_period, + translate_legacy_policy, +) + + +def _session_and_catalog(): + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + session = Session(engine) + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + first = Parameter( + name="gov.example.rate", + tax_benefit_model_version=version, + ) + second = Parameter( + name="gov.example.amount", + tax_benefit_model_version=version, + ) + session.add_all([first, second]) + session.commit() + return engine, session, model, version, first, second + + +def _snapshot(**changes) -> LegacyPolicySnapshot: + fields = { + "country_id": "us", + "legacy_policy_id": 42, + "label": "Presentation only", + "api_version": "1.0.0", + "policy_json": { + "gov.example.rate": {"2026": 0.2}, + "gov.example.amount": {"2026-01-01.2026-12-31": 100}, + }, + "source_policy_hash": "legacy/base64+hash=", + } + fields.update(changes) + return LegacyPolicySnapshot.model_validate(fields) + + +def test_year_day_and_explicit_range_periods_are_inclusive_utc() -> None: + assert parse_legacy_period("2026") == ( + datetime(2026, 1, 1, tzinfo=timezone.utc), + datetime(2026, 12, 31, tzinfo=timezone.utc), + ) + assert parse_legacy_period("2026-03-02") == ( + datetime(2026, 3, 2, tzinfo=timezone.utc), + datetime(2026, 3, 2, tzinfo=timezone.utc), + ) + assert parse_legacy_period("2026-02-01.2026-02-28") == ( + datetime(2026, 2, 1, tzinfo=timezone.utc), + datetime(2026, 2, 28, tzinfo=timezone.utc), + ) + + +def test_translation_resolves_paths_and_excludes_legacy_identity_and_label() -> None: + engine, session, model, version, first, second = _session_and_catalog() + try: + translated = translate_legacy_policy( + session, + _snapshot(), + running_policyengine_version="5.2.0", + country_package_versions={"us": "1.0.0"}, + ) + + assert translated.tax_benefit_model_id == model.id + assert translated.tax_benefit_model_version_id == version.id + assert {value.parameter_id for value in translated.parameter_values} == { + first.id, + second.id, + } + assert "label" not in type(translated).model_fields + assert "legacy_policy_id" not in type(translated).model_fields + finally: + session.close() + engine.dispose() + + +def test_label_does_not_change_translated_core_content() -> None: + engine, session, _model, _version, _first, _second = _session_and_catalog() + try: + first = translate_legacy_policy( + session, + _snapshot(label="First"), + running_policyengine_version="5.2.0", + country_package_versions={"us": "1.0.0"}, + ) + second = translate_legacy_policy( + session, + _snapshot(label="Second", legacy_policy_id=43), + running_policyengine_version="5.2.0", + country_package_versions={"us": "1.0.0"}, + ) + assert first == second + finally: + session.close() + engine.dispose() + + +@pytest.mark.parametrize( + "policy_json", + [ + {"gov.missing": {"2026": 1}}, + {"gov.example.rate": 1}, + {"gov.example.rate": {"not-a-period": 1}}, + { + "gov.example.rate": { + "2026": 1, + "2026-01-01.2026-12-31": 2, + } + }, + ], +) +def test_missing_paths_malformed_periods_and_conflicts_fail( + policy_json: dict[str, object], +) -> None: + engine, session, _model, _version, _first, _second = _session_and_catalog() + try: + with pytest.raises(LegacyPolicyTranslationError): + translate_legacy_policy( + session, + _snapshot(policy_json=policy_json), + running_policyengine_version="5.2.0", + country_package_versions={"us": "1.0.0"}, + ) + finally: + session.close() + engine.dispose() + + +def test_country_package_version_must_match_running_release() -> None: + engine, session, _model, _version, _first, _second = _session_and_catalog() + try: + with pytest.raises(LegacyPolicyTranslationError, match="api_version"): + translate_legacy_policy( + session, + _snapshot(api_version="0.9.0"), + running_policyengine_version="5.2.0", + country_package_versions={"us": "1.0.0"}, + ) + finally: + session.close() + engine.dispose() + + +@pytest.mark.parametrize( + "changes", + [ + {"policy_json": ["not", "an", "object"]}, + {"source_policy_hash": ""}, + {"legacy_policy_id": -1}, + {"policy_json": {"gov.example": {"2026": float("nan")}}}, + ], +) +def test_snapshot_rejects_incomplete_or_non_json_committed_fields(changes) -> None: + with pytest.raises(ValidationError): + _snapshot(**changes) + + +def test_reverse_legacy_range_is_rejected() -> None: + with pytest.raises(LegacyPolicyTranslationError, match="ends before"): + parse_legacy_period("2026-12-31.2026-01-01") + + +def test_unknown_policyengine_version_never_falls_back() -> None: + engine, session, _model, _version, _first, _second = _session_and_catalog() + try: + with pytest.raises(Exception, match="running PolicyEngine.py"): + translate_legacy_policy( + session, + _snapshot(), + running_policyengine_version="4.0.0", + country_package_versions={"us": "1.0.0"}, + ) + finally: + session.close() + engine.dispose() diff --git a/tests/unit/v2/test_policy_migration_qualification.py b/tests/unit/v2/test_policy_migration_qualification.py new file mode 100644 index 000000000..170953b28 --- /dev/null +++ b/tests/unit/v2/test_policy_migration_qualification.py @@ -0,0 +1,131 @@ +"""Tests for read-only v2 policy migration qualification.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +import pytest + +from policyengine_api.data.v2 import policy_migration_qualification as qualification +from policyengine_api.data.v2.settings import ( + V2_MIGRATION_DATABASE_URL, + V2_SUPABASE_ENVIRONMENT, + V2_SUPABASE_PROJECT_REF, +) + + +ENVIRONMENT = { + V2_MIGRATION_DATABASE_URL: ( + "postgresql+psycopg://migration:test-password@db." + "abcdefghijklmnopqrst.supabase.co/postgres?sslmode=require" + ), + V2_SUPABASE_PROJECT_REF: "abcdefghijklmnopqrst", + V2_SUPABASE_ENVIRONMENT: "staging", +} + + +class FakeTransaction: + def __init__(self) -> None: + self.rolled_back = False + + def rollback(self) -> None: + self.rolled_back = True + + +class FakeConnection: + def __init__(self, counts: tuple[int, int, int]) -> None: + self._counts = iter(counts) + self.transaction = FakeTransaction() + self.statements: list[str] = [] + + def begin(self) -> FakeTransaction: + return self.transaction + + def exec_driver_sql(self, statement: str) -> None: + self.statements.append(statement) + + def scalar(self, _statement: object) -> int: + return next(self._counts) + + +class FakeEngine: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + self.disposed = False + + @contextmanager + def connect(self) -> Iterator[FakeConnection]: + yield self.connection + + def dispose(self) -> None: + self.disposed = True + + +def test_empty_target_is_qualified_in_a_rolled_back_read_only_transaction() -> None: + connection = FakeConnection((0, 0, 0)) + engine = FakeEngine(connection) + + evidence = qualification.qualify_policy_migration_target( + ENVIRONMENT, + engine_builder=lambda _settings: engine, + ) + + assert evidence.as_dict() == { + "outcome": "ok", + "environment": "staging", + "project_ref": "abcdefghijklmnopqrst", + "counts": { + "policies": 0, + "policy_parameter_values": 0, + "user_policies": 0, + }, + } + assert connection.statements == ["SET TRANSACTION READ ONLY"] + assert connection.transaction.rolled_back + assert engine.disposed + + +@pytest.mark.parametrize( + "counts", + [ + (1, 0, 0), + (0, 1, 0), + (0, 0, 1), + (1, 2, 3), + ], +) +def test_retained_policy_data_stops_without_committing( + counts: tuple[int, int, int], +) -> None: + connection = FakeConnection(counts) + engine = FakeEngine(connection) + + with pytest.raises( + qualification.RetainedPolicyDataError, + match="migration stopped without modifying data", + ) as raised: + qualification.qualify_policy_migration_target( + ENVIRONMENT, + engine_builder=lambda _settings: engine, + ) + + assert raised.value.counts == qualification.PolicyDataCounts(*counts) + assert connection.transaction.rolled_back + assert engine.disposed + + +def test_main_redacts_unexpected_errors( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def fail() -> None: + raise RuntimeError("postgresql://user:secret@private-host/database") + + monkeypatch.setattr(qualification, "qualify_policy_migration_target", fail) + + assert qualification.main() == 1 + captured = capsys.readouterr() + assert "qualification failed unexpectedly" in captured.err + assert "secret" not in captured.err + assert "private-host" not in captured.err diff --git a/tests/unit/v2/test_policy_persistence_statements.py b/tests/unit/v2/test_policy_persistence_statements.py new file mode 100644 index 000000000..f51716d86 --- /dev/null +++ b/tests/unit/v2/test_policy_persistence_statements.py @@ -0,0 +1,26 @@ +"""Unit checks for conflict-aware policy insertion statements.""" + +from __future__ import annotations + +from sqlalchemy.dialects import postgresql + +from policyengine_api.data.v2.policies import persistence + + +def test_policy_insert_uses_the_content_identity_constraint_and_returning() -> None: + source = persistence._insert_policy.__code__.co_consts + statement_text = " ".join(str(value) for value in source) + + assert "uq_policies_canonicalization_content_hash" in statement_text + + # Compile a representative statement through the same PostgreSQL dialect + # construct to prove this module does not use a read-before-write insert. + statement = ( + persistence.insert(persistence.Policy) + .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") + .returning(persistence.Policy.id) + ) + compiled = str(statement.compile(dialect=postgresql.dialect())) + assert "ON CONFLICT ON CONSTRAINT" in compiled + assert "DO NOTHING" in compiled + assert "RETURNING policies.id" in compiled diff --git a/tests/unit/v2/test_policy_query.py b/tests/unit/v2/test_policy_query.py new file mode 100644 index 000000000..86c4c72b8 --- /dev/null +++ b/tests/unit/v2/test_policy_query.py @@ -0,0 +1,188 @@ +"""Country-scoped complete policy read tests.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from sqlmodel import Session, create_engine +import pytest + +from policyengine_api.data.v2.models import ( + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + V2_METADATA, +) +from policyengine_api.data.v2.policies.query import ( + PolicyNotFoundError, + list_policies, + read_policy, +) + + +def _stored_policies(): + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + session = Session(engine) + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + alpha = Parameter(name="gov.alpha", tax_benefit_model_version=version) + zeta = Parameter(name="gov.zeta", tax_benefit_model_version=version) + created = datetime(2026, 1, 1, tzinfo=timezone.utc) + first = Policy( + id=UUID("00000000-0000-0000-0000-000000000010"), + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="1" * 64, + created_at=created, + updated_at=created, + ) + second = Policy( + id=UUID("00000000-0000-0000-0000-000000000020"), + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="2" * 64, + created_at=created + timedelta(seconds=1), + updated_at=created + timedelta(seconds=1), + ) + other_country = Policy( + id=UUID("00000000-0000-0000-0000-000000000030"), + country_id="uk", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="3" * 64, + created_at=created + timedelta(seconds=2), + updated_at=created + timedelta(seconds=2), + ) + session.add_all( + [ + first, + second, + other_country, + ParameterValue( + id=UUID("00000000-0000-0000-0000-000000000200"), + policy=first, + parameter=zeta, + value_json=2, + start_date=created, + ), + ParameterValue( + id=UUID("00000000-0000-0000-0000-000000000100"), + policy=first, + parameter=alpha, + value_json=1, + start_date=created + timedelta(days=1), + ), + ParameterValue( + id=UUID("00000000-0000-0000-0000-000000000090"), + policy=first, + parameter=alpha, + value_json=0, + start_date=created, + ), + ] + ) + session.commit() + return engine, session, model, first, second, other_country + + +def test_detail_joins_parameter_names_and_orders_complete_values() -> None: + engine, session, _model, first, _second, _other = _stored_policies() + try: + result = read_policy(session, country_id="us", policy_id=first.id) + + assert result.id == first.id + assert result.created_at == first.created_at + assert [value.parameter_name for value in result.parameter_values] == [ + "gov.alpha", + "gov.alpha", + "gov.zeta", + ] + assert [value.value for value in result.parameter_values] == [0, 1, 2] + finally: + session.close() + engine.dispose() + + +def test_detail_uses_country_as_part_of_resource_identity() -> None: + engine, session, _model, first, _second, _other = _stored_policies() + try: + with pytest.raises(PolicyNotFoundError): + read_policy(session, country_id="uk", policy_id=first.id) + finally: + session.close() + engine.dispose() + + +def test_empty_policy_has_an_empty_nested_collection() -> None: + engine, session, _model, _first, second, _other = _stored_policies() + try: + assert ( + read_policy( + session, + country_id="us", + policy_id=second.id, + ).parameter_values + == () + ) + finally: + session.close() + engine.dispose() + + +def test_collection_filters_orders_paginates_and_returns_complete_items() -> None: + engine, session, model, first, second, _other = _stored_policies() + try: + first_page = list_policies( + session, + country_id="us", + tax_benefit_model_id=model.id, + offset=0, + limit=1, + ) + second_page = list_policies( + session, + country_id="us", + tax_benefit_model_id=model.id, + offset=1, + limit=1, + ) + + assert [item.id for item in first_page.items] == [first.id] + assert len(first_page.items[0].parameter_values) == 3 + assert first_page.has_more is True + assert (first_page.offset, first_page.limit) == (0, 1) + assert [item.id for item in second_page.items] == [second.id] + assert second_page.has_more is False + assert all(item.country_id == "us" for item in first_page.items) + finally: + session.close() + engine.dispose() + + +def test_model_filter_is_exact() -> None: + engine, session, _model, _first, _second, _other = _stored_policies() + try: + result = list_policies( + session, + country_id="us", + tax_benefit_model_id=UUID("ffffffff-ffff-ffff-ffff-ffffffffffff"), + ) + assert result.items == () + assert result.has_more is False + finally: + session.close() + engine.dispose() diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py new file mode 100644 index 000000000..dc711c5a3 --- /dev/null +++ b/tests/unit/v2/test_policy_routes.py @@ -0,0 +1,421 @@ +"""Native FastAPI contract tests for immutable v2 policies.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import UUID, uuid4 + +from fastapi.testclient import TestClient +from flask import Flask, jsonify +import pytest +from sqlalchemy.exc import OperationalError, TimeoutError + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.data.v2.policies.catalog import PolicyCatalogValidationError +from policyengine_api.data.v2.policies.persistence import ( + PolicyContentHashCollisionError, + PolicyPersistenceIntegrityError, +) +from policyengine_api.data.v2.policies.query import ( + PolicyNotFoundError, + PolicyPage, + PolicyParameterValueRead, + PolicyRead, +) +from policyengine_api.data.v2.policies.service import NativePolicyCreation +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) + + +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +MODEL_ID = UUID("00000000-0000-0000-0000-000000000020") +MODEL_VERSION_ID = UUID("00000000-0000-0000-0000-000000000030") +PARAMETER_ID = UUID("00000000-0000-0000-0000-000000000040") +VALUE_ID = UUID("00000000-0000-0000-0000-000000000050") +CREATED_AT = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _policy_read(*, policy_id: UUID = POLICY_ID, country_id: str = "us") -> PolicyRead: + return PolicyRead( + id=policy_id, + country_id=country_id, + tax_benefit_model_id=MODEL_ID, + tax_benefit_model_version_id=MODEL_VERSION_ID, + created_at=CREATED_AT, + updated_at=CREATED_AT + timedelta(seconds=1), + parameter_values=( + PolicyParameterValueRead( + id=VALUE_ID, + parameter_id=PARAMETER_ID, + parameter_name="gov.example.rate", + value={"rate": 0.2}, + start_date=CREATED_AT, + end_date=None, + ), + ), + ) + + +class FakePolicyService: + def __init__(self) -> None: + self.created = True + self.error: Exception | None = None + self.calls: list[tuple[str, object]] = [] + + def _raise(self) -> None: + if self.error is not None: + raise self.error + + def create_policy(self, command) -> NativePolicyCreation: + self.calls.append(("create_policy", command)) + self._raise() + return NativePolicyCreation(item=_policy_read(), created=self.created) + + def get_policy(self, **filters) -> PolicyRead: + self.calls.append(("get_policy", filters)) + self._raise() + return _policy_read(policy_id=filters["policy_id"]) + + def list_policies(self, **filters) -> PolicyPage: + self.calls.append(("list_policies", filters)) + self._raise() + return PolicyPage( + items=(_policy_read(),), + offset=filters["offset"], + limit=filters["limit"], + has_more=False, + ) + + +def _client( + service: FakePolicyService, +) -> tuple[TestClient, dict[str, int]]: + flask_calls = {"count": 0} + flask_app = Flask(__name__) + + @flask_app.route("/", methods=["GET", "POST", "PATCH", "DELETE"]) + def fallback(resource: str): + flask_calls["count"] += 1 + return jsonify({"source": "flask", "resource": resource}) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=lambda: {}, + v2_policy_service_factory=lambda: service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + return ( + TestClient( + create_asgi_app( + flask_app, + dependencies=dependencies, + route_settings=settings, + ), + raise_server_exceptions=False, + ), + flask_calls, + ) + + +def _body(**changes) -> dict[str, object]: + body: dict[str, object] = { + "country_id": "us", + "tax_benefit_model_id": str(MODEL_ID), + "parameter_values": [ + { + "parameter_id": str(PARAMETER_ID), + "value": {"rate": 0.2}, + "start_date": "2026-01-01T00:00:00Z", + } + ], + } + body.update(changes) + return body + + +def test_create_returns_201_for_new_and_200_for_deduplicated_content() -> None: + service = FakePolicyService() + client, flask_calls = _client(service) + + created = client.post( + "/v2/policies?country_id=US&policyengine_version=5.2.0", + json=_body(), + ) + service.created = False + deduplicated = client.post("/v2/policies?country_id=us", json=_body()) + + assert created.status_code == 201 + assert deduplicated.status_code == 200 + assert created.json() == deduplicated.json() + assert created.json() == { + "status": "ok", + "message": None, + "result": { + "item": { + "id": str(POLICY_ID), + "country_id": "us", + "tax_benefit_model_id": str(MODEL_ID), + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:01Z", + "parameter_values": [ + { + "id": str(VALUE_ID), + "parameter_id": str(PARAMETER_ID), + "parameter_name": "gov.example.rate", + "value": {"rate": 0.2}, + "start_date": "2026-01-01T00:00:00Z", + "end_date": None, + } + ], + } + }, + } + first_command = service.calls[0][1] + assert first_command.country_id == "us" + assert first_command.policyengine_version == "5.2.0" + assert flask_calls["count"] == 0 + + +def test_create_rejects_country_mismatch_and_core_presentation_fields() -> None: + service = FakePolicyService() + client, _flask_calls = _client(service) + + mismatch = client.post("/v2/policies?country_id=uk", json=_body()) + named = client.post( + "/v2/policies?country_id=us", + json={**_body(), "name": "Not core content", "description": "No"}, + ) + + assert mismatch.status_code == 400 + assert named.status_code == 422 + assert service.calls == [] + + +def test_create_rejects_unknown_duplicate_and_oversized_input_before_service() -> None: + service = FakePolicyService() + client, _flask_calls = _client(service) + + unknown = client.post( + "/v2/policies?country_id=us&search=rate", + json=_body(), + ) + duplicate = client.post( + "/v2/policies?country_id=us&country_id=uk", + json=_body(), + ) + excessive_values = client.post( + "/v2/policies?country_id=us", + json=_body( + parameter_values=[ + { + "parameter_id": str(uuid4()), + "value": index, + "start_date": "2026-01-01T00:00:00Z", + } + for index in range(1_001) + ] + ), + ) + oversized = client.post( + "/v2/policies?country_id=us", + content=b'{"country_id":"us","padding":"' + b"x" * 1_048_576 + b'"}', + headers={"content-type": "application/json"}, + ) + + assert unknown.status_code == 422 + assert duplicate.status_code == 422 + assert excessive_values.status_code == 422 + assert oversized.status_code == 413 + assert oversized.json()["status"] == "error" + assert service.calls == [] + + +def test_detail_is_country_scoped_and_returns_typed_not_found() -> None: + service = FakePolicyService() + client, flask_calls = _client(service) + + response = client.get(f"/v2/policies/{POLICY_ID}?country_id=US") + service.error = PolicyNotFoundError("policy was not found") + missing = client.get(f"/v2/policies/{uuid4()}?country_id=uk") + + assert response.status_code == 200 + assert ( + response.json()["result"]["item"]["parameter_values"][0]["parameter_name"] + == "gov.example.rate" + ) + assert service.calls[0] == ( + "get_policy", + {"country_id": "us", "policy_id": POLICY_ID}, + ) + assert missing.status_code == 404 + assert missing.json() == {"status": "error", "message": "policy was not found"} + assert flask_calls["count"] == 0 + + +def test_list_passes_exact_model_filter_and_canonical_pagination() -> None: + service = FakePolicyService() + client, _flask_calls = _client(service) + + response = client.get( + f"/v2/policies?country_id=us&tax_benefit_model_id={MODEL_ID}&offset=2&limit=3" + ) + search = client.get("/v2/policies?country_id=us&search=rate") + excessive = client.get("/v2/policies?country_id=us&limit=501") + + assert response.status_code == 200 + assert response.json()["result"]["offset"] == 2 + assert response.json()["result"]["limit"] == 3 + assert service.calls == [ + ( + "list_policies", + { + "country_id": "us", + "tax_benefit_model_id": MODEL_ID, + "offset": 2, + "limit": 3, + }, + ) + ] + assert search.status_code == 422 + assert excessive.status_code == 422 + + +@pytest.mark.parametrize( + ("error", "status", "message"), + [ + (PolicyCatalogValidationError("bad model"), 400, "bad model"), + (MetadataCatalogVersionNotFoundError("absent catalog"), 404, "absent"), + ( + PolicyContentHashCollisionError("database statement secret"), + 409, + "conflicts", + ), + ( + PolicyPersistenceIntegrityError("database statement secret"), + 500, + "integrity", + ), + (V2ConfigurationError("postgresql://secret"), 503, "unavailable"), + ( + MetadataCatalogUnavailableError("database statement secret"), + 503, + "unavailable", + ), + ( + OperationalError("statement secret", {}, Exception("credential")), + 503, + "unavailable", + ), + (TimeoutError("pool timeout"), 503, "unavailable"), + (RuntimeError("credential and policy value"), 500, "operation failed"), + ], +) +def test_policy_failures_map_to_secret_safe_typed_errors( + error: Exception, + status: int, + message: str, +) -> None: + service = FakePolicyService() + service.error = error + client, _flask_calls = _client(service) + + response = client.get(f"/v2/policies/{POLICY_ID}?country_id=us") + + assert response.status_code == status + assert response.json()["status"] == "error" + assert message in response.json()["message"].lower() + assert "secret" not in response.text + assert "credential" not in response.text + + +@pytest.mark.parametrize("method", ["put", "patch", "delete"]) +def test_core_policy_mutations_are_not_exposed(method: str) -> None: + service = FakePolicyService() + client, flask_calls = _client(service) + + response = client.request( + method.upper(), + f"/v2/policies/{POLICY_ID}?country_id=us", + json={}, + ) + + assert response.status_code == 405 + assert response.json()["status"] == "error" + assert service.calls == [] + assert flask_calls["count"] == 0 + + +def test_openapi_publishes_request_query_response_and_error_contracts() -> None: + service = FakePolicyService() + client, _flask_calls = _client(service) + schema = client.get("/v2/openapi.json").json() + + assert set(path for path in schema["paths"] if path.startswith("/v2/policies")) == { + "/v2/policies", + "/v2/policies/{policy_id}", + } + collection = schema["paths"]["/v2/policies"]["get"] + parameters = {item["name"]: item for item in collection["parameters"]} + assert parameters["country_id"]["required"] is True + assert parameters["offset"]["schema"]["default"] == 0 + assert parameters["limit"]["schema"]["default"] == 100 + assert parameters["limit"]["schema"]["maximum"] == 500 + assert parameters["tax_benefit_model_id"]["schema"]["anyOf"][0]["format"] == ( + "uuid" + ) + + create = schema["paths"]["/v2/policies"]["post"] + create_parameters = {item["name"]: item for item in create["parameters"]} + assert create_parameters["country_id"]["required"] is True + assert create_parameters["policyengine_version"]["required"] is False + assert {"200", "201", "400", "404", "409", "413", "422", "500", "503"} <= ( + set(create["responses"]) + ) + request_ref = create["requestBody"]["content"]["application/json"]["schema"]["$ref"] + request_schema = schema["components"]["schemas"][request_ref.rsplit("/", 1)[-1]] + assert request_schema["additionalProperties"] is False + parameter_values = request_schema["properties"]["parameter_values"] + assert parameter_values["maxItems"] == 1000 + item_schema = schema["components"]["schemas"]["PolicyItem"] + assert set(item_schema["required"]) == { + "id", + "country_id", + "tax_benefit_model_id", + "created_at", + "updated_at", + "parameter_values", + } + assert "name" not in item_schema["properties"] + assert "description" not in item_schema["properties"] + + +def test_native_policy_routes_do_not_use_cloud_sql_or_flask( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_cloud_sql(): + raise AssertionError("Cloud SQL must not be selected") + + monkeypatch.setattr( + "policyengine_api.data.orm.get_v1_session_factory", + reject_cloud_sql, + ) + service = FakePolicyService() + client, flask_calls = _client(service) + + assert client.get(f"/v2/policies/{POLICY_ID}?country_id=us").status_code == 200 + assert service.calls[0][0] == "get_policy" + assert flask_calls["count"] == 0 diff --git a/tests/unit/v2/test_settings.py b/tests/unit/v2/test_settings.py index 020aa4f13..f093fd275 100644 --- a/tests/unit/v2/test_settings.py +++ b/tests/unit/v2/test_settings.py @@ -207,6 +207,20 @@ def test_pooler_username_can_identify_the_configured_supabase_project() -> None: assert settings.connection.url.username == f"data-writer.{PROJECT_REF}" +def test_runtime_rejects_transaction_pooling_without_session_timeouts() -> None: + with pytest.raises(V2ConfigurationError, match="session mode on port 5432"): + load_v2_runtime_database_settings( + { + **TARGET_ENVIRONMENT, + V2_RUNTIME_DATABASE_URL: ( + f"postgresql+psycopg://runtime.{PROJECT_REF}:password@" + "aws-0-us-east-2.pooler.supabase.com:6543/" + "postgres?sslmode=require" + ), + } + ) + + def test_supabase_database_name_must_be_postgres() -> None: with pytest.raises(V2ConfigurationError, match="configured Supabase database"): load_v2_runtime_database_settings( diff --git a/tests/unit/v2/test_user_policy_legacy.py b/tests/unit/v2/test_user_policy_legacy.py new file mode 100644 index 000000000..9edbc816f --- /dev/null +++ b/tests/unit/v2/test_user_policy_legacy.py @@ -0,0 +1,191 @@ +"""Legacy saved-policy fingerprint and projection command tests.""" + +from __future__ import annotations + +from unittest.mock import MagicMock +from uuid import UUID + +import pytest + +from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy +from policyengine_api.data.v2.user_policies.legacy import ( + USER_POLICY_FINGERPRINT_VERSION, + LegacyUserPolicyIntegrityError, + LegacyUserPolicySnapshot, + _apply_existing_mapping, + fingerprint_legacy_user_policy, +) + + +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") + + +def _snapshot(**changes) -> LegacyUserPolicySnapshot: + values = { + "country_id": "us", + "legacy_user_policy_id": 10, + "reform_id": 2, + "reform_label": "Reform", + "baseline_id": 1, + "baseline_label": "Current law", + "user_id": "auth0|one", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + values.update(changes) + return LegacyUserPolicySnapshot.model_validate(values) + + +def test_complete_row_fingerprint_is_deterministic_and_sha256() -> None: + first = _snapshot() + reordered = LegacyUserPolicySnapshot.model_validate( + dict(reversed(list(first.model_dump().items()))) + ) + + assert fingerprint_legacy_user_policy(first) == fingerprint_legacy_user_policy( + reordered + ) + assert len(fingerprint_legacy_user_policy(first)) == 64 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("reform_label", "Updated"), + ("baseline_label", "Baseline updated"), + ("year", "2027"), + ("geography", "ca"), + ("dataset", None), + ("number_of_provisions", 4), + ("api_version", "2.0.0"), + ("added_date", 3), + ("updated_date", 4), + ("budgetary_impact", "100"), + ("type", "reform"), + ], +) +def test_every_mutable_or_v1_only_field_changes_the_fingerprint( + field: str, + value: object, +) -> None: + assert fingerprint_legacy_user_policy(_snapshot()) != ( + fingerprint_legacy_user_policy(_snapshot(**{field: value})) + ) + + +def test_snapshot_rejects_unknown_or_unbounded_fields() -> None: + with pytest.raises(ValueError): + _snapshot(unexpected="value") + with pytest.raises(ValueError): + _snapshot(user_id="x" * 256) + + +def _apply_update( + *, + changed_fields: frozenset[str], + stored_revision: int = 0, + source_revision: int = 1, + stored_fingerprint: str = "a" * 64, + fingerprint: str = "b" * 64, +): + association = UserPolicy( + country_id="us", + user_id="auth0|one", + policy_id=POLICY_ID, + name="Native name", + description="Native description", + ) + mapping = LegacyUserPolicyMapping( + country_id="us", + legacy_user_policy_id=10, + user_policy_id=association.id, + last_applied_source_revision=stored_revision, + fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, + fingerprint_sha256=stored_fingerprint, + ) + session = MagicMock() + session.exec.return_value.one_or_none.return_value = association + result = _apply_existing_mapping( + session, + mapping=mapping, + snapshot=_snapshot(reform_label="Legacy rename", year="2027"), + fingerprint=fingerprint, + policy_id=POLICY_ID, + changed_fields=changed_fields, + source_revision=source_revision, + ) + return association, mapping, result + + +def test_v1_only_update_advances_fingerprint_without_changing_presentation() -> None: + association, mapping, result = _apply_update( + changed_fields=frozenset({"year", "updated_date"}) + ) + + assert result.association_updated is False + assert association.name == "Native name" + assert association.description == "Native description" + assert mapping.fingerprint_sha256 == "b" * 64 + assert mapping.last_applied_source_revision == 1 + + +def test_reform_label_update_changes_only_name() -> None: + association, mapping, result = _apply_update( + changed_fields=frozenset({"reform_label", "updated_date"}) + ) + + assert result.association_updated is True + assert association.name == "Legacy rename" + assert association.description == "Native description" + assert mapping.fingerprint_sha256 == "b" * 64 + assert mapping.last_applied_source_revision == 1 + + +def test_same_revision_and_fingerprint_is_an_idempotent_replay() -> None: + association, mapping, result = _apply_update( + changed_fields=frozenset({"reform_label"}), + stored_revision=3, + source_revision=3, + stored_fingerprint="b" * 64, + ) + + assert result.association_updated is False + assert association.name == "Native name" + assert mapping.last_applied_source_revision == 3 + + +def test_same_revision_with_different_fingerprint_is_rejected() -> None: + with pytest.raises(LegacyUserPolicyIntegrityError, match="revision conflicts"): + _apply_update( + changed_fields=frozenset({"reform_label"}), + stored_revision=3, + source_revision=3, + ) + + +def test_stale_revision_is_an_idempotent_no_op() -> None: + association, mapping, result = _apply_update( + changed_fields=frozenset({"reform_label"}), + stored_revision=3, + source_revision=2, + ) + + assert result.association_updated is False + assert association.name == "Native name" + assert mapping.last_applied_source_revision == 3 + + +def test_revision_with_an_unapplied_predecessor_is_rejected() -> None: + with pytest.raises(LegacyUserPolicyIntegrityError, match="unapplied predecessor"): + _apply_update( + changed_fields=frozenset({"reform_label"}), + stored_revision=1, + source_revision=3, + ) diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py new file mode 100644 index 000000000..aca9699b8 --- /dev/null +++ b/tests/unit/v2/test_user_policy_routes.py @@ -0,0 +1,416 @@ +"""Native FastAPI contract tests for v2 user-policy associations.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import UUID, uuid4 + +from fastapi.testclient import TestClient +from flask import Flask, jsonify +import pytest +from sqlalchemy.exc import OperationalError, SQLAlchemyError, TimeoutError + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.data.v2.user_policies.persistence import ( + AssociationCountryConflictError, + AssociationPolicyNotFoundError, +) +from policyengine_api.data.v2.user_policies.query import ( + UserPolicyNotFoundError, + UserPolicyPage, + UserPolicyRead, +) +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) + + +ASSOCIATION_ID = UUID("00000000-0000-0000-0000-000000000060") +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +CREATED_AT = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _association_read( + *, + association_id: UUID = ASSOCIATION_ID, + country_id: str = "us", + user_id: str = "auth0|caller", + policy_id: UUID = POLICY_ID, + name: str | None = "Saved reform", + description: str | None = "Personal note", +) -> UserPolicyRead: + return UserPolicyRead( + id=association_id, + country_id=country_id, + user_id=user_id, + policy_id=policy_id, + name=name, + description=description, + created_at=CREATED_AT, + updated_at=CREATED_AT + timedelta(seconds=1), + ) + + +class FakeUserPolicyService: + def __init__(self) -> None: + self.error: Exception | None = None + self.calls: list[tuple[str, object]] = [] + self.next_item = _association_read() + + def _raise(self) -> None: + if self.error is not None: + raise self.error + + def create_user_policy(self, command) -> UserPolicyRead: + self.calls.append(("create_user_policy", command)) + self._raise() + return self.next_item + + def get_user_policy(self, **identity) -> UserPolicyRead: + self.calls.append(("get_user_policy", identity)) + self._raise() + return self.next_item + + def list_user_policies(self, **filters) -> UserPolicyPage: + self.calls.append(("list_user_policies", filters)) + self._raise() + return UserPolicyPage( + items=(self.next_item,), + offset=filters["offset"], + limit=filters["limit"], + has_more=False, + ) + + def patch_user_policy(self, **changes) -> UserPolicyRead: + self.calls.append(("patch_user_policy", changes)) + self._raise() + command = changes["command"] + name = command.name if "name" in command.model_fields_set else "Saved reform" + description = ( + command.description + if "description" in command.model_fields_set + else "Personal note" + ) + return _association_read(name=name, description=description) + + def delete_user_policy(self, **identity) -> None: + self.calls.append(("delete_user_policy", identity)) + self._raise() + + +def _client( + service: FakeUserPolicyService, +) -> tuple[TestClient, dict[str, int]]: + flask_calls = {"count": 0} + flask_app = Flask(__name__) + + @flask_app.route("/", methods=["GET", "POST", "PATCH", "DELETE"]) + def fallback(resource: str): + flask_calls["count"] += 1 + return jsonify({"source": "flask", "resource": resource}) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=lambda: {}, + v2_user_policy_service_factory=lambda: service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + return ( + TestClient( + create_asgi_app( + flask_app, + dependencies=dependencies, + route_settings=settings, + ), + raise_server_exceptions=False, + ), + flask_calls, + ) + + +def _body(**changes) -> dict[str, object]: + body: dict[str, object] = { + "country_id": "us", + "user_id": "auth0|caller", + "policy_id": str(POLICY_ID), + "name": "Saved reform", + "description": "Personal note", + } + body.update(changes) + return body + + +def test_create_returns_complete_distinct_association_contract() -> None: + service = FakeUserPolicyService() + client, flask_calls = _client(service) + + response = client.post("/v2/user-policies?country_id=US", json=_body()) + + assert response.status_code == 201 + assert response.json() == { + "status": "ok", + "message": None, + "result": { + "item": { + "id": str(ASSOCIATION_ID), + "country_id": "us", + "user_id": "auth0|caller", + "policy_id": str(POLICY_ID), + "name": "Saved reform", + "description": "Personal note", + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:01Z", + } + }, + } + command = service.calls[0][1] + assert command.user_id == "auth0|caller" + assert command.policy_id == POLICY_ID + assert flask_calls["count"] == 0 + + +def test_repeated_create_calls_service_twice_without_link_deduplication() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + + first = client.post("/v2/user-policies?country_id=us", json=_body()) + service.next_item = _association_read(association_id=uuid4()) + second = client.post("/v2/user-policies?country_id=us", json=_body()) + + assert first.status_code == second.status_code == 201 + assert first.json()["result"]["item"]["id"] != second.json()["result"]["item"]["id"] + assert [call[0] for call in service.calls] == [ + "create_user_policy", + "create_user_policy", + ] + + +def test_create_rejects_country_mismatch_and_bounded_fields() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + + mismatch = client.post("/v2/user-policies?country_id=uk", json=_body()) + empty_user = client.post( + "/v2/user-policies?country_id=us", + json=_body(user_id=" "), + ) + long_name = client.post( + "/v2/user-policies?country_id=us", + json=_body(name="x" * 256), + ) + + assert mismatch.status_code == 400 + assert empty_user.status_code == 422 + assert long_name.status_code == 422 + assert service.calls == [] + + +def test_detail_and_list_use_country_user_policy_and_pagination_filters() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + + detail = client.get(f"/v2/user-policies/{ASSOCIATION_ID}?country_id=US") + page = client.get( + "/v2/user-policies?country_id=us&user_id=auth0%7Ccaller" + f"&policy_id={POLICY_ID}&offset=2&limit=3" + ) + + assert detail.status_code == 200 + assert page.status_code == 200 + assert service.calls == [ + ( + "get_user_policy", + {"country_id": "us", "association_id": ASSOCIATION_ID}, + ), + ( + "list_user_policies", + { + "country_id": "us", + "user_id": "auth0|caller", + "policy_id": POLICY_ID, + "offset": 2, + "limit": 3, + }, + ), + ] + assert page.json()["result"] == { + "items": [detail.json()["result"]["item"]], + "offset": 2, + "limit": 3, + "has_more": False, + } + + +def test_collection_rejects_missing_unknown_duplicate_and_invalid_queries() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + + responses = [ + client.get("/v2/user-policies?country_id=us"), + client.get("/v2/user-policies?country_id=us&user_id=u&search=reform"), + client.get("/v2/user-policies?country_id=us&country_id=uk&user_id=u"), + client.get("/v2/user-policies?country_id=us&user_id=u&limit=501"), + ] + + assert [response.status_code for response in responses] == [422] * 4 + assert all(response.json()["status"] == "error" for response in responses) + assert service.calls == [] + + +def test_patch_supports_explicit_null_and_rejects_identity_or_empty_changes() -> None: + service = FakeUserPolicyService() + client, flask_calls = _client(service) + + cleared = client.patch( + f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us", + json={"name": None}, + ) + identity = client.patch( + f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us", + json={"policy_id": str(uuid4())}, + ) + empty = client.patch( + f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us", + json={}, + ) + + assert cleared.status_code == 200 + assert cleared.json()["result"]["item"]["name"] is None + assert identity.status_code == 422 + assert empty.status_code == 422 + assert [call[0] for call in service.calls] == ["patch_user_policy"] + assert flask_calls["count"] == 0 + + +def test_delete_returns_no_content_and_uses_country_scoped_identity() -> None: + service = FakeUserPolicyService() + client, flask_calls = _client(service) + + response = client.delete(f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us") + + assert response.status_code == 204 + assert response.content == b"" + assert service.calls == [ + ( + "delete_user_policy", + {"country_id": "us", "association_id": ASSOCIATION_ID}, + ) + ] + assert flask_calls["count"] == 0 + + +@pytest.mark.parametrize( + ("error", "status", "message"), + [ + (AssociationCountryConflictError("different country"), 400, "country"), + (AssociationPolicyNotFoundError("policy was not found"), 404, "not found"), + (UserPolicyNotFoundError("association was not found"), 404, "not found"), + (V2ConfigurationError("postgresql://secret"), 503, "unavailable"), + ( + OperationalError("statement secret", {}, Exception("credential")), + 503, + "unavailable", + ), + (TimeoutError("pool timeout"), 503, "unavailable"), + (SQLAlchemyError("statement secret"), 503, "unavailable"), + (RuntimeError("credential and caller data"), 500, "operation failed"), + ], +) +def test_association_failures_map_to_secret_safe_typed_errors( + error: Exception, + status: int, + message: str, +) -> None: + service = FakeUserPolicyService() + service.error = error + client, _flask_calls = _client(service) + + response = client.get(f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us") + + assert response.status_code == status + assert response.json()["status"] == "error" + assert message in response.json()["message"].lower() + assert "secret" not in response.text + assert "credential" not in response.text + + +def test_openapi_publishes_complete_no_auth_association_contracts() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + schema = client.get("/v2/openapi.json").json() + + assert { + path for path in schema["paths"] if path.startswith("/v2/user-policies") + } == { + "/v2/user-policies", + "/v2/user-policies/{association_id}", + } + collection = schema["paths"]["/v2/user-policies"]["get"] + parameters = {item["name"]: item for item in collection["parameters"]} + assert parameters["country_id"]["required"] is True + assert parameters["user_id"]["required"] is True + assert "Unverified caller-supplied" in parameters["user_id"]["description"] + assert parameters["limit"]["schema"]["maximum"] == 500 + assert set(schema["paths"]["/v2/user-policies"]) == {"get", "post"} + assert set(schema["paths"]["/v2/user-policies/{association_id}"]) == { + "get", + "patch", + "delete", + } + + for path, method in ( + ("/v2/user-policies", "post"), + ("/v2/user-policies", "get"), + ("/v2/user-policies/{association_id}", "get"), + ("/v2/user-policies/{association_id}", "patch"), + ("/v2/user-policies/{association_id}", "delete"), + ): + operation = schema["paths"][path][method] + assert "security" not in operation + assert {"400", "404", "409", "422", "500", "503"} <= set(operation["responses"]) + + item_schema = schema["components"]["schemas"]["UserPolicyItem"] + assert set(item_schema["required"]) == { + "id", + "country_id", + "user_id", + "policy_id", + "name", + "description", + "created_at", + "updated_at", + } + patch = schema["paths"]["/v2/user-policies/{association_id}"]["patch"] + patch_ref = patch["requestBody"]["content"]["application/json"]["schema"]["$ref"] + patch_schema = schema["components"]["schemas"][patch_ref.rsplit("/", 1)[-1]] + assert patch_schema["additionalProperties"] is False + assert set(patch_schema["properties"]) == {"name", "description"} + + +def test_native_association_routes_do_not_use_cloud_sql_or_flask( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_cloud_sql(): + raise AssertionError("Cloud SQL must not be selected") + + monkeypatch.setattr( + "policyengine_api.data.orm.get_v1_session_factory", + reject_cloud_sql, + ) + service = FakeUserPolicyService() + client, flask_calls = _client(service) + + response = client.get(f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us") + + assert response.status_code == 200 + assert service.calls[0][0] == "get_user_policy" + assert flask_calls["count"] == 0 diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py new file mode 100644 index 000000000..7cd422eb4 --- /dev/null +++ b/tests/unit/v2/test_user_policy_service.py @@ -0,0 +1,213 @@ +"""Service tests for native v2 user-policy associations.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +import sqlalchemy as sa +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session, create_engine, select + +from policyengine_api.data.v2.models import ( + LegacyUserPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + UserPolicy, + V2_METADATA, +) +from policyengine_api.data.v2.user_policies.persistence import ( + AssociationCountryConflictError, + AssociationPolicyNotFoundError, +) +from policyengine_api.data.v2.user_policies.query import UserPolicyNotFoundError +from policyengine_api.data.v2.user_policies.schemas import ( + UserPolicyCreateCommand, + UserPolicyPatchCommand, +) +from policyengine_api.data.v2.user_policies.service import V2UserPolicyService + + +@pytest.fixture +def association_store(): + engine = create_engine("sqlite://") + + @sa.event.listens_for(engine, "connect") + def enable_foreign_keys(dbapi_connection, _connection_record) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + V2_METADATA.create_all(engine) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + with sessions.begin() as session: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.example.rate", + tax_benefit_model_version=version, + ) + policy = Policy( + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="a" * 64, + ) + value = ParameterValue( + parameter=parameter, + policy=policy, + value_json=0.2, + start_date=datetime(2026, 1, 1, tzinfo=timezone.utc), + ) + session.add(value) + session.flush() + identity = (policy.id, value.id) + + yield V2UserPolicyService(sessions), sessions, identity + engine.dispose() + + +def _command(policy_id, **changes) -> UserPolicyCreateCommand: + values = { + "country_id": "us", + "user_id": "auth0|caller", + "policy_id": policy_id, + "name": "Saved reform", + "description": "Personal note", + } + values.update(changes) + return UserPolicyCreateCommand.model_validate(values) + + +def test_create_allows_distinct_duplicate_links_and_unverified_user( + association_store, +) -> None: + service, sessions, (policy_id, _value_id) = association_store + + first = service.create_user_policy(_command(policy_id)) + second = service.create_user_policy(_command(policy_id, name="Second save")) + + assert first.id != second.id + assert first.user_id == "auth0|caller" + assert second.policy_id == policy_id + with sessions() as session: + assert len(session.exec(select(UserPolicy)).all()) == 2 + + +def test_create_rejects_missing_policy_and_country_conflict( + association_store, +) -> None: + service, _sessions, (policy_id, _value_id) = association_store + + with pytest.raises(AssociationPolicyNotFoundError): + service.create_user_policy(_command("00000000-0000-0000-0000-000000000099")) + with pytest.raises(AssociationCountryConflictError): + service.create_user_policy(_command(policy_id, country_id="uk")) + + +def test_detail_list_filter_and_pagination_are_country_scoped( + association_store, +) -> None: + service, _sessions, (policy_id, _value_id) = association_store + first = service.create_user_policy(_command(policy_id, name="First")) + service.create_user_policy(_command(policy_id, name="Second")) + service.create_user_policy( + _command(policy_id, user_id="another-user", name="Other") + ) + + detail = service.get_user_policy( + country_id="us", + association_id=first.id, + ) + page = service.list_user_policies( + country_id="us", + user_id="auth0|caller", + policy_id=policy_id, + limit=1, + ) + second_page = service.list_user_policies( + country_id="us", + user_id="auth0|caller", + offset=1, + limit=1, + ) + + assert detail.id == first.id + assert [item.name for item in page.items] == ["First"] + assert page.has_more is True + assert [item.name for item in second_page.items] == ["Second"] + with pytest.raises(UserPolicyNotFoundError): + service.get_user_policy(country_id="uk", association_id=first.id) + + +def test_patch_changes_only_supplied_fields_and_supports_null_clearing( + association_store, +) -> None: + service, _sessions, (policy_id, _value_id) = association_store + created = service.create_user_policy(_command(policy_id)) + + renamed = service.patch_user_policy( + country_id="us", + association_id=created.id, + command=UserPolicyPatchCommand(name="Renamed"), + ) + cleared = service.patch_user_policy( + country_id="us", + association_id=created.id, + command=UserPolicyPatchCommand(description=None), + ) + + assert renamed.name == "Renamed" + assert renamed.description == "Personal note" + assert cleared.name == "Renamed" + assert cleared.description is None + assert cleared.updated_at >= created.updated_at + assert (cleared.country_id, cleared.user_id, cleared.policy_id) == ( + "us", + "auth0|caller", + policy_id, + ) + + +def test_delete_removes_mapping_but_preserves_policy_and_parameter_value( + association_store, +) -> None: + service, sessions, (policy_id, value_id) = association_store + created = service.create_user_policy(_command(policy_id)) + with sessions.begin() as session: + mapping = LegacyUserPolicyMapping( + country_id="us", + legacy_user_policy_id=42, + user_policy_id=created.id, + fingerprint_version=1, + fingerprint_sha256="b" * 64, + ) + session.add(mapping) + session.flush() + mapping_id = mapping.id + + service.delete_user_policy(country_id="us", association_id=created.id) + + with sessions() as session: + assert session.get(UserPolicy, created.id) is None + assert session.get(LegacyUserPolicyMapping, mapping_id) is None + assert session.get(Policy, policy_id) is not None + assert session.get(ParameterValue, value_id) is not None + + +def test_patch_command_rejects_empty_and_immutable_fields() -> None: + with pytest.raises(ValueError): + UserPolicyPatchCommand() + with pytest.raises(ValueError): + UserPolicyPatchCommand.model_validate( + {"policy_id": "00000000-0000-0000-0000-000000000001"} + ) diff --git a/uv.lock b/uv.lock index de2dde78d..886b8790b 100644 --- a/uv.lock +++ b/uv.lock @@ -2544,7 +2544,7 @@ models = [ [[package]] name = "policyengine-api" -version = "3.50.0" +version = "3.52.0" source = { editable = "." } dependencies = [ { name = "a2wsgi" }, From 4cbe956ab177bde1f36d7a6ac5f4da16588bd105 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:18:49 +0400 Subject: [PATCH 02/18] Add Stage 10 changelog fragment --- changelog.d/stage-10.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/stage-10.added.md diff --git a/changelog.d/stage-10.added.md b/changelog.d/stage-10.added.md new file mode 100644 index 000000000..50acd514a --- /dev/null +++ b/changelog.d/stage-10.added.md @@ -0,0 +1 @@ +Add API v2 policies and user-policy associations with immediate API v1 mutation mirroring. From b8b2d8d2488b64a999db6d86a6106f86462df83b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:18:51 +0400 Subject: [PATCH 03/18] Restore UUID user policy ownership mapping --- docs/migration/stage-10-v2-policies.md | 13 +- ...34023a728f_map_legacy_users_to_v2_uuids.py | 111 ++++++++++++++++++ .../data/v2/catalog/publication.py | 2 +- policyengine_api/data/v2/models/__init__.py | 2 + .../data/v2/models/associations.py | 7 +- .../data/v2/models/policy_mappings.py | 29 ++++- policyengine_api/data/v2/models/users.py | 11 +- .../data/v2/user_policies/api_schemas.py | 2 +- .../data/v2/user_policies/legacy.py | 80 ++++++++++++- .../data/v2/user_policies/persistence.py | 11 +- .../data/v2/user_policies/query.py | 6 +- .../data/v2/user_policies/service.py | 2 +- .../fastapi_routes/v2_user_policies.py | 18 ++- policyengine_api/query_parameters.py | 18 ++- .../contract/test_policy_v2_compatibility.py | 8 +- .../integration/test_alembic_v2_lifecycle.py | 24 +++- .../test_v1_user_policy_dual_write.py | 33 ++++++ .../test_v2_user_policy_mirroring.py | 103 +++++++++++++++- tests/unit/test_query_parameters.py | 5 +- tests/unit/v2/test_alembic_v2.py | 30 ++++- tests/unit/v2/test_import_side_effects.py | 2 +- tests/unit/v2/test_model_persistence.py | 45 ++++++- tests/unit/v2/test_models.py | 32 ++++- tests/unit/v2/test_user_policy_legacy.py | 4 +- tests/unit/v2/test_user_policy_routes.py | 32 ++--- tests/unit/v2/test_user_policy_service.py | 38 ++++-- 26 files changed, 588 insertions(+), 80 deletions(-) create mode 100644 migrations/v2/versions/af34023a728f_map_legacy_users_to_v2_uuids.py diff --git a/docs/migration/stage-10-v2-policies.md b/docs/migration/stage-10-v2-policies.md index 43880a8f3..134ec7af3 100644 --- a/docs/migration/stage-10-v2-policies.md +++ b/docs/migration/stage-10-v2-policies.md @@ -14,7 +14,7 @@ integer identifiers remain in Cloud SQL throughout this stage. Supavisor session mode on port 5432. The runtime rejects transaction-pooling endpoints on port 6543 because they cannot apply the per-session statement timeout. -3. Before applying revision `711ec2f0a5a5`, run the dormant-table qualification +3. Before applying the Phase 10 revisions beginning with `711ec2f0a5a5`, run the dormant-table qualification against the exact migration target: ```bash @@ -49,7 +49,7 @@ uv run alembic -c alembic-v2.ini upgrade head Run each Alembic `check` command against the same corresponding target and confirm no metadata/schema difference. The relevant generated revisions are -`3d6e8f553ca5` for MySQL and `c21c4a807a49` for PostgreSQL. +`3d6e8f553ca5` for MySQL and `af34023a728f` for the current PostgreSQL head. ## Activation @@ -58,6 +58,15 @@ Supabase connection. The routes are registered as preview resources; `ROUTE_IMPL_POLICY=fastapi_native` declares them operational for deployment readiness validation without moving v1 routes away from Flask. +Native user-policy requests use an existing `users.id` UUID. V1 saved-policy +mirroring keeps the opaque Cloud SQL user identifier out of that foreign-key +column: `legacy_user_mappings` maps the exact v1 string one-to-one to a v2 user +UUID. First use creates a minimal v2 user and mapping in the same Supabase +transaction as the association; later saves and retries reuse that UUID. +`first_name`, `last_name`, and `email` are null for these transition-created +users because the v1 saved-policy record does not supply them. This stage does +not add or infer Auth0 identifiers. + Keep the initial v1 settings explicit: ```text diff --git a/migrations/v2/versions/af34023a728f_map_legacy_users_to_v2_uuids.py b/migrations/v2/versions/af34023a728f_map_legacy_users_to_v2_uuids.py new file mode 100644 index 000000000..43d0cf4b2 --- /dev/null +++ b/migrations/v2/versions/af34023a728f_map_legacy_users_to_v2_uuids.py @@ -0,0 +1,111 @@ +"""map legacy users to v2 uuids + +Revision ID: af34023a728f +Revises: c21c4a807a49 +Create Date: 2026-09-01 22:06:28.086723 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "af34023a728f" +down_revision: Union[str, None] = "c21c4a807a49" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "legacy_user_mappings", + sa.Column( + "legacy_user_id", + sqlmodel.sql.sqltypes.AutoString(length=255), + nullable=False, + ), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name=op.f("fk_legacy_user_mappings_user_id_users"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("legacy_user_id", name=op.f("pk_legacy_user_mappings")), + sa.UniqueConstraint("user_id", name="uq_legacy_user_mappings_user_id"), + ) + op.create_index( + op.f("ix_legacy_user_mappings_user_id"), + "legacy_user_mappings", + ["user_id"], + unique=False, + ) + op.alter_column( + "user_policies", + "user_id", + existing_type=sa.VARCHAR(length=255), + type_=sa.Uuid(), + existing_nullable=False, + # Post-generation correction: PostgreSQL requires an explicit + # cast when restoring the UUID association foreign key. + postgresql_using="user_id::uuid", + ) + op.create_foreign_key( + op.f("fk_user_policies_user_id_users"), + "user_policies", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.alter_column( + "users", "first_name", existing_type=sa.VARCHAR(length=255), nullable=True + ) + op.alter_column( + "users", "last_name", existing_type=sa.VARCHAR(length=255), nullable=True + ) + op.alter_column( + "users", "email", existing_type=sa.VARCHAR(length=320), nullable=True + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "users", "email", existing_type=sa.VARCHAR(length=320), nullable=False + ) + op.alter_column( + "users", "last_name", existing_type=sa.VARCHAR(length=255), nullable=False + ) + op.alter_column( + "users", "first_name", existing_type=sa.VARCHAR(length=255), nullable=False + ) + op.drop_constraint( + op.f("fk_user_policies_user_id_users"), "user_policies", type_="foreignkey" + ) + op.alter_column( + "user_policies", + "user_id", + existing_type=sa.Uuid(), + type_=sa.VARCHAR(length=255), + existing_nullable=False, + # Post-generation correction: PostgreSQL requires an explicit + # cast for the reversible UUID-to-text downgrade. + postgresql_using="user_id::text", + ) + op.drop_index( + op.f("ix_legacy_user_mappings_user_id"), table_name="legacy_user_mappings" + ) + op.drop_table("legacy_user_mappings") + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 9b028cd92..2f12581a0 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -32,7 +32,7 @@ ) -EXPECTED_ALEMBIC_REVISION = "c21c4a807a49" +EXPECTED_ALEMBIC_REVISION = "af34023a728f" # Stable application-defined PostgreSQL lock ID shared by all v2 catalog publishers. PUBLICATION_ADVISORY_LOCK_KEY = 8_629_020_026_090_001 diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 253073562..4459e8c11 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -53,6 +53,7 @@ ) from policyengine_api.data.v2.models.policy_mappings import ( # noqa: E402 LegacyPolicyMapping, + LegacyUserMapping, LegacyUserPolicyMapping, ) from policyengine_api.data.v2.models.reports import ( # noqa: E402 @@ -99,6 +100,7 @@ "IntraDecileImpact", "LocalAuthorityImpact", "LegacyPolicyMapping", + "LegacyUserMapping", "LegacyUserPolicyMapping", "OutputStatus", "Parameter", diff --git a/policyengine_api/data/v2/models/associations.py b/policyengine_api/data/v2/models/associations.py index 0581defe0..38ba0ce77 100644 --- a/policyengine_api/data/v2/models/associations.py +++ b/policyengine_api/data/v2/models/associations.py @@ -79,12 +79,17 @@ class UserPolicy(TimestampedModel, table=True): ), ) - user_id: str = Field(max_length=255, index=True) + user_id: UUID = Field( + foreign_key="users.id", + ondelete="CASCADE", + index=True, + ) policy_id: UUID = Field(index=True) country_id: str = Field(max_length=2) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) + user: User = Relationship(back_populates="policy_associations") policy: Policy = Relationship(back_populates="user_associations") legacy_mapping: Optional["LegacyUserPolicyMapping"] = Relationship( back_populates="association", diff --git a/policyengine_api/data/v2/models/policy_mappings.py b/policyengine_api/data/v2/models/policy_mappings.py index b08d04b8c..7cdbc91b9 100644 --- a/policyengine_api/data/v2/models/policy_mappings.py +++ b/policyengine_api/data/v2/models/policy_mappings.py @@ -1,18 +1,43 @@ """Durable source-identity mappings for immediate v1 policy mirroring.""" +from datetime import datetime from typing import TYPE_CHECKING from uuid import UUID import sqlalchemy as sa -from sqlmodel import Field, Relationship +from sqlmodel import Field, Relationship, SQLModel -from policyengine_api.data.v2.models.base import TimestampedModel +from policyengine_api.data.v2.models.base import TimestampedModel, utc_now if TYPE_CHECKING: from policyengine_api.data.v2.models.associations import UserPolicy from policyengine_api.data.v2.models.policies import Policy +class LegacyUserMapping(SQLModel, table=True): + """Map one exact v1 user identifier to one v2 user UUID.""" + + __tablename__ = "legacy_user_mappings" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + name="uq_legacy_user_mappings_user_id", + ), + ) + + legacy_user_id: str = Field(primary_key=True, max_length=255) + user_id: UUID = Field( + foreign_key="users.id", + ondelete="RESTRICT", + index=True, + ) + created_at: datetime = Field( + default_factory=utc_now, + sa_type=sa.DateTime(timezone=True), + sa_column_kwargs={"server_default": sa.func.now()}, + ) + + class LegacyPolicyMapping(TimestampedModel, table=True): """Map one country-scoped v1 policy ID to deduplicated v2 content.""" diff --git a/policyengine_api/data/v2/models/users.py b/policyengine_api/data/v2/models/users.py index 6a5a62ae9..ca85c46ce 100644 --- a/policyengine_api/data/v2/models/users.py +++ b/policyengine_api/data/v2/models/users.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from policyengine_api.data.v2.models.associations import ( UserHouseholdAssociation, + UserPolicy, UserReportAssociation, UserSimulationAssociation, ) @@ -26,9 +27,9 @@ class User(IdentifiedModel, table=True): ), ) - first_name: str = Field(max_length=255) - last_name: str = Field(max_length=255) - email: str = Field(max_length=320, index=True) + first_name: str | None = Field(default=None, max_length=255) + last_name: str | None = Field(default=None, max_length=255) + email: str | None = Field(default=None, max_length=320, index=True) primary_country: str = Field(max_length=2) reports: list["Report"] = Relationship(back_populates="user") @@ -44,3 +45,7 @@ class User(IdentifiedModel, table=True): back_populates="user", cascade_delete=True, ) + policy_associations: list["UserPolicy"] = Relationship( + back_populates="user", + cascade_delete=True, + ) diff --git a/policyengine_api/data/v2/user_policies/api_schemas.py b/policyengine_api/data/v2/user_policies/api_schemas.py index a1b23ccbd..a1467108d 100644 --- a/policyengine_api/data/v2/user_policies/api_schemas.py +++ b/policyengine_api/data/v2/user_policies/api_schemas.py @@ -97,7 +97,7 @@ class UserPolicyErrorResponse(StrictUserPolicyAPIModel): }, 404: { "model": UserPolicyErrorResponse, - "description": "The selected policy or association does not exist.", + "description": "The selected user, policy, or association does not exist.", }, 409: { "model": UserPolicyErrorResponse, diff --git a/policyengine_api/data/v2/user_policies/legacy.py b/policyengine_api/data/v2/user_policies/legacy.py index 4321969e5..88ec6fec4 100644 --- a/policyengine_api/data/v2/user_policies/legacy.py +++ b/policyengine_api/data/v2/user_policies/legacy.py @@ -12,7 +12,12 @@ from sqlalchemy.dialects.postgresql import insert from sqlmodel import Session, select -from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy +from policyengine_api.data.v2.models import ( + LegacyUserMapping, + LegacyUserPolicyMapping, + User, + UserPolicy, +) from policyengine_api.data.v2.models.base import utc_now from policyengine_api.data.v2.policies.legacy import ( LegacyPolicySnapshot, @@ -20,7 +25,7 @@ ) from policyengine_api.data.v2.policies.schemas import StrictPolicyCommand from policyengine_api.data.v2.user_policies.schemas import UserPolicyCreateCommand -from policyengine_api.query_parameters import CountryId, UserId +from policyengine_api.query_parameters import CountryId, LegacyUserId USER_POLICY_FINGERPRINT_VERSION = 1 @@ -39,7 +44,7 @@ class LegacyUserPolicySnapshot(StrictPolicyCommand): reform_label: Annotated[str, Field(max_length=255)] | None = None baseline_id: Annotated[int, Field(ge=0)] baseline_label: Annotated[str, Field(max_length=255)] | None = None - user_id: UserId + user_id: LegacyUserId year: Annotated[str, Field(max_length=32)] geography: Annotated[str, Field(max_length=255)] dataset: Annotated[str, Field(max_length=255)] | None = None @@ -80,19 +85,75 @@ def fingerprint_legacy_user_policy(snapshot: LegacyUserPolicySnapshot) -> str: def project_legacy_user_policy( snapshot: LegacyUserPolicySnapshot, *, + user_id: UUID, policy_id: UUID, ) -> UserPolicyCreateCommand: """Map v1 presentation data onto an association, never core policy content.""" return UserPolicyCreateCommand( country_id=snapshot.country_id, - user_id=snapshot.user_id, + user_id=user_id, policy_id=policy_id, name=snapshot.reform_label, description=None, ) +def _legacy_user_mapping( + session: Session, + legacy_user_id: str, + *, + lock: bool, +) -> LegacyUserMapping | None: + statement = select(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == legacy_user_id + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def resolve_legacy_user_id( + session: Session, + *, + legacy_user_id: str, + primary_country: str, +) -> UUID: + """Return one durable v2 UUID for an exact opaque v1 user identifier.""" + + existing = _legacy_user_mapping(session, legacy_user_id, lock=True) + if existing is not None: + if session.get(User, existing.user_id) is None: + raise LegacyUserPolicyIntegrityError( + "legacy user mapping has no referenced v2 user" + ) + return existing.user_id + + user = User(primary_country=primary_country) + session.add(user) + session.flush() + inserted_user_id = session.execute( + insert(LegacyUserMapping) + .values( + legacy_user_id=legacy_user_id, + user_id=user.id, + ) + .on_conflict_do_nothing(index_elements=[LegacyUserMapping.legacy_user_id]) + .returning(LegacyUserMapping.user_id) + ).scalar_one_or_none() + if inserted_user_id is not None: + return inserted_user_id + + session.delete(user) + session.flush() + concurrent = _legacy_user_mapping(session, legacy_user_id, lock=False) + if concurrent is None or session.get(User, concurrent.user_id) is None: + raise LegacyUserPolicyIntegrityError( + "legacy user mapping conflict did not resolve to a v2 user" + ) + return concurrent.user_id + + def _mapping( session: Session, snapshot: LegacyUserPolicySnapshot, @@ -131,6 +192,7 @@ def _apply_existing_mapping( mapping: LegacyUserPolicyMapping, snapshot: LegacyUserPolicySnapshot, fingerprint: str, + user_id: UUID, policy_id: UUID, changed_fields: frozenset[str], source_revision: int, @@ -139,7 +201,7 @@ def _apply_existing_mapping( if ( association.policy_id != policy_id or association.country_id != snapshot.country_id - or association.user_id != snapshot.user_id + or association.user_id != user_id ): raise LegacyUserPolicyIntegrityError( "legacy user-policy mapping conflicts with immutable association fields" @@ -216,6 +278,11 @@ def persist_legacy_user_policy( "saved policy does not reference the supplied reform snapshot" ) policy_result = persist_legacy_policy(session, reform_snapshot) + user_id = resolve_legacy_user_id( + session, + legacy_user_id=snapshot.user_id, + primary_country=snapshot.country_id, + ) fingerprint = fingerprint_legacy_user_policy(snapshot) existing = _mapping(session, snapshot, lock=True) if existing is not None: @@ -224,6 +291,7 @@ def persist_legacy_user_policy( mapping=existing, snapshot=snapshot, fingerprint=fingerprint, + user_id=user_id, policy_id=policy_result.policy_id, changed_fields=changed_fields, source_revision=source_revision, @@ -231,6 +299,7 @@ def persist_legacy_user_policy( projection = project_legacy_user_policy( snapshot, + user_id=user_id, policy_id=policy_result.policy_id, ) association = UserPolicy(**projection.model_dump()) @@ -272,6 +341,7 @@ def persist_legacy_user_policy( mapping=concurrent, snapshot=snapshot, fingerprint=fingerprint, + user_id=user_id, policy_id=policy_result.policy_id, changed_fields=changed_fields, source_revision=source_revision, diff --git a/policyengine_api/data/v2/user_policies/persistence.py b/policyengine_api/data/v2/user_policies/persistence.py index 55364f9bd..adf22a99b 100644 --- a/policyengine_api/data/v2/user_policies/persistence.py +++ b/policyengine_api/data/v2/user_policies/persistence.py @@ -6,7 +6,7 @@ from sqlmodel import Session, select -from policyengine_api.data.v2.models import Policy, UserPolicy +from policyengine_api.data.v2.models import Policy, User, UserPolicy from policyengine_api.data.v2.models.base import utc_now from policyengine_api.data.v2.user_policies.query import ( UserPolicyRead, @@ -23,6 +23,10 @@ class AssociationPolicyNotFoundError(LookupError): """Raised when an association references an unknown policy UUID.""" +class AssociationUserNotFoundError(LookupError): + """Raised when an association references an unknown v2 user UUID.""" + + class AssociationCountryConflictError(ValueError): """Raised when an association and its referenced policy differ by country.""" @@ -31,7 +35,10 @@ def create_user_policy( session: Session, command: UserPolicyCreateCommand, ) -> UserPolicyRead: - """Create one independently identified association after policy validation.""" + """Create one independently identified association after link validation.""" + + if session.get(User, command.user_id) is None: + raise AssociationUserNotFoundError(f"user {command.user_id} was not found") policy = session.exec( select(Policy).where(Policy.id == command.policy_id) diff --git a/policyengine_api/data/v2/user_policies/query.py b/policyengine_api/data/v2/user_policies/query.py index ec18e01f9..6c910bf10 100644 --- a/policyengine_api/data/v2/user_policies/query.py +++ b/policyengine_api/data/v2/user_policies/query.py @@ -19,7 +19,7 @@ class UserPolicyNotFoundError(LookupError): class UserPolicyRead: id: UUID country_id: str - user_id: str + user_id: UUID policy_id: UUID name: str | None description: str | None @@ -88,12 +88,12 @@ def list_user_policies( session: Session, *, country_id: str, - user_id: str, + user_id: UUID, policy_id: UUID | None = None, offset: int = 0, limit: int = 100, ) -> UserPolicyPage: - """Read one deterministic bounded page for a supplied user identifier.""" + """Read one deterministic bounded page for a v2 user UUID.""" statement = select(UserPolicy).where( UserPolicy.country_id == country_id, diff --git a/policyengine_api/data/v2/user_policies/service.py b/policyengine_api/data/v2/user_policies/service.py index 3562c7d01..fc42eae4d 100644 --- a/policyengine_api/data/v2/user_policies/service.py +++ b/policyengine_api/data/v2/user_policies/service.py @@ -60,7 +60,7 @@ def list_user_policies( self, *, country_id: str, - user_id: str, + user_id: UUID, policy_id: UUID | None = None, offset: int = 0, limit: int = 100, diff --git a/policyengine_api/fastapi_routes/v2_user_policies.py b/policyengine_api/fastapi_routes/v2_user_policies.py index 7f2f9c4f3..16b23fb8f 100644 --- a/policyengine_api/fastapi_routes/v2_user_policies.py +++ b/policyengine_api/fastapi_routes/v2_user_policies.py @@ -24,6 +24,7 @@ from policyengine_api.data.v2.user_policies.persistence import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, + AssociationUserNotFoundError, ) from policyengine_api.data.v2.user_policies.query import UserPolicyNotFoundError from policyengine_api.fastapi_routes.dependencies import ( @@ -64,7 +65,11 @@ def _association_operation( return operation() except AssociationCountryConflictError as error: return user_policy_error_response(400, str(error)) - except (AssociationPolicyNotFoundError, UserPolicyNotFoundError) as error: + except ( + AssociationPolicyNotFoundError, + AssociationUserNotFoundError, + UserPolicyNotFoundError, + ) as error: return user_policy_error_response(404, str(error)) except (V2ConfigurationError, SQLAlchemyError): return user_policy_error_response( @@ -91,9 +96,10 @@ def build_v2_user_policy_router( status_code=201, summary="Create a user-policy association", description=( - "Creates a saved association for an unverified caller-supplied " - "user identifier. This operation performs no authentication or " - "authorization check." + "Creates a saved association for an existing v2 user UUID. The " + "UUID identifies a database row; this operation does not prove " + "that the caller controls that user and performs no authentication " + "or authorization check." ), ) def create_user_policy( @@ -139,8 +145,8 @@ def read() -> UserPolicyDetailResponse: response_model=UserPolicyPageResponse, summary="List user-policy associations", description=( - "Filters by an unverified caller-supplied user identifier. A match " - "is not an authentication or authorization decision." + "Filters by a v2 user UUID. A match is not proof of caller control " + "and is not an authentication or authorization decision." ), ) def get_user_policies( diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index a889b79f7..a1fd12abf 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -32,11 +32,13 @@ def normalize_country_id(value: Any) -> Any: return value.lower() if isinstance(value, str) else value -def validate_user_id(value: str) -> str: - """Reject an empty or whitespace-only caller-supplied identifier.""" +def validate_legacy_user_id(value: str) -> str: + """Reject an empty or whitespace-only legacy user identifier.""" if not value.strip(): - raise ValueError("user_id must contain at least one non-whitespace character") + raise ValueError( + "legacy user_id must contain at least one non-whitespace character" + ) return value @@ -61,13 +63,17 @@ def validate_user_id(value: str) -> str: ] ResourceId = Annotated[UUID, Field(description="Exact resource UUID")] UserId = Annotated[ + UUID, + Field(description="V2 user UUID; does not prove caller control"), +] +LegacyUserId = Annotated[ str, Field( min_length=1, max_length=MAXIMUM_USER_ID_LENGTH, - description="Unverified caller-supplied user identifier", + description="Exact opaque v1 user identifier", ), - AfterValidator(validate_user_id), + AfterValidator(validate_legacy_user_id), ] @@ -111,7 +117,7 @@ class PolicyCollectionQuery(CountryQuery, PaginationQuery): class UserPolicyCollectionQuery(CountryQuery, PaginationQuery): - """Query contract for one caller-supplied user's policy associations.""" + """Query contract for one v2 user's policy associations.""" user_id: UserId policy_id: ResourceId | None = None diff --git a/tests/contract/test_policy_v2_compatibility.py b/tests/contract/test_policy_v2_compatibility.py index 8962623c8..2ebbd5e81 100644 --- a/tests/contract/test_policy_v2_compatibility.py +++ b/tests/contract/test_policy_v2_compatibility.py @@ -15,6 +15,7 @@ POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +USER_ID = UUID("00000000-0000-0000-0000-000000000070") def test_app_v2_reform_label_maps_to_association_name_only() -> None: @@ -37,10 +38,15 @@ def test_app_v2_reform_label_maps_to_association_name_only() -> None: type=None, ) - projection = project_legacy_user_policy(snapshot, policy_id=POLICY_ID) + projection = project_legacy_user_policy( + snapshot, + user_id=USER_ID, + policy_id=POLICY_ID, + ) assert projection.name == "User-visible app label" assert projection.description is None + assert projection.user_id == USER_ID assert projection.policy_id == POLICY_ID assert "name" not in Policy.__table__.c assert "description" not in Policy.__table__.c diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index fe5358147..6978d5187 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -23,8 +23,8 @@ BASELINE_REVISION = "f5ef4347cb2a" -PREVIOUS_HEAD_REVISION = "711ec2f0a5a5" -HEAD_REVISION = "c21c4a807a49" +PREVIOUS_HEAD_REVISION = "c21c4a807a49" +HEAD_REVISION = "af34023a728f" V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) @@ -89,6 +89,7 @@ def _assert_head(engine) -> None: ] assert { "legacy_policy_mappings", + "legacy_user_mappings", "legacy_user_policy_mappings", } <= set(inspect(engine).get_table_names(schema="public")) @@ -112,13 +113,26 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: with engine.connect() as connection: context = MigrationContext.configure(connection) assert context.get_current_revision() == PREVIOUS_HEAD_REVISION - assert "last_applied_source_revision" not in { - column["name"] + assert "legacy_user_mappings" not in inspect(engine).get_table_names( + schema="public" + ) + user_policy_id = next( + column for column in inspect(engine).get_columns( - "legacy_user_policy_mappings", + "user_policies", schema="public", ) + if column["name"] == "user_id" + ) + assert isinstance(user_policy_id["type"], sa.String) + user_columns = { + column["name"]: column + for column in inspect(engine).get_columns("users", schema="public") } + assert all( + not user_columns[field_name]["nullable"] + for field_name in ("first_name", "last_name", "email") + ) command.upgrade(config, "head") command.check(config) diff --git a/tests/integration/test_v1_user_policy_dual_write.py b/tests/integration/test_v1_user_policy_dual_write.py index cc7335a0a..643623e2d 100644 --- a/tests/integration/test_v1_user_policy_dual_write.py +++ b/tests/integration/test_v1_user_policy_dual_write.py @@ -17,12 +17,14 @@ ) from policyengine_api.data.v2.models import ( LegacyPolicyMapping, + LegacyUserMapping, LegacyUserPolicyMapping, Parameter, ParameterValue, Policy, TaxBenefitModel, TaxBenefitModelVersion, + User, UserPolicy, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL @@ -141,6 +143,20 @@ def _cleanup(engine, model_id) -> None: if model_id is None: return with engine.begin() as connection: + user_ids = ( + connection.execute( + select(LegacyUserMapping.user_id).where( + LegacyUserMapping.legacy_user_id == "auth0|cross-database" + ) + ) + .scalars() + .all() + ) + connection.execute( + delete(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == "auth0|cross-database" + ) + ) policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) association_ids = select(UserPolicy.id).where( UserPolicy.policy_id.in_(policy_ids) @@ -153,6 +169,8 @@ def _cleanup(engine, model_id) -> None: connection.execute( delete(UserPolicy).where(UserPolicy.policy_id.in_(policy_ids)) ) + if user_ids: + connection.execute(delete(User).where(User.id.in_(user_ids))) connection.execute( delete(LegacyPolicyMapping).where( LegacyPolicyMapping.policy_id.in_(policy_ids) @@ -245,6 +263,9 @@ def test_create_update_and_v1_only_change_mirror_one_association() -> None: association = session.get(UserPolicy, first.association_id) assert association.name == "Renamed" assert association.description is None + user_mapping = session.scalar(select(LegacyUserMapping)) + assert association.user_id == user_mapping.user_id + assert session.get(User, user_mapping.user_id).primary_country == "us" assert ( session.scalar( select(func.count()).select_from(LegacyUserPolicyMapping) @@ -297,6 +318,10 @@ def test_failure_after_cloud_commit_and_identical_create_retry_are_idempotent() event = session.scalar(select(UserPolicyMirrorEvent)) assert event.processed_at is None with v2_sessions() as session: + assert session.scalar(select(func.count()).select_from(User)) == 0 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 0 + ) assert ( session.scalar( select(func.count()).select_from(LegacyUserPolicyMapping) @@ -328,6 +353,10 @@ def test_failure_after_cloud_commit_and_identical_create_retry_are_idempotent() with v2_sessions() as session: mapping = session.scalar(select(LegacyUserPolicyMapping)) assert mapping.last_applied_source_revision == 2 + assert session.scalar(select(func.count()).select_from(User)) == 1 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 1 + ) finally: _cleanup(v2_engine, model_id) v1_engine.dispose() @@ -378,6 +407,10 @@ def test_destination_commit_replays_when_source_processing_marker_is_missing() - assert event.processed_at is not None with v2_sessions() as session: assert session.scalar(select(func.count()).select_from(UserPolicy)) == 1 + assert session.scalar(select(func.count()).select_from(User)) == 1 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 1 + ) mapping = session.scalar(select(LegacyUserPolicyMapping)) assert mapping.last_applied_source_revision == 1 finally: diff --git a/tests/integration/test_v2_user_policy_mirroring.py b/tests/integration/test_v2_user_policy_mirroring.py index 582e8f2c1..58b31a8a4 100644 --- a/tests/integration/test_v2_user_policy_mirroring.py +++ b/tests/integration/test_v2_user_policy_mirroring.py @@ -2,7 +2,10 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor import os +from threading import Barrier +from uuid import UUID, uuid4 import pytest from sqlalchemy import create_engine, delete, func, select @@ -16,12 +19,14 @@ ) from policyengine_api.data.v2.models import ( LegacyPolicyMapping, + LegacyUserMapping, LegacyUserPolicyMapping, Parameter, ParameterValue, Policy, TaxBenefitModel, TaxBenefitModelVersion, + User, UserPolicy, ) from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot @@ -30,6 +35,7 @@ LegacyUserPolicySnapshot, fingerprint_legacy_user_policy, persist_legacy_user_policy, + resolve_legacy_user_id, ) @@ -107,6 +113,21 @@ def _cleanup(engine, model_id) -> None: if model_id is None: return with engine.begin() as connection: + legacy_user_ids = ("auth0|one", "legacy-user-two") + user_ids = ( + connection.execute( + select(LegacyUserMapping.user_id).where( + LegacyUserMapping.legacy_user_id.in_(legacy_user_ids) + ) + ) + .scalars() + .all() + ) + connection.execute( + delete(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id.in_(legacy_user_ids) + ) + ) policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) association_ids = select(UserPolicy.id).where( UserPolicy.policy_id.in_(policy_ids) @@ -119,6 +140,8 @@ def _cleanup(engine, model_id) -> None: connection.execute( delete(UserPolicy).where(UserPolicy.policy_id.in_(policy_ids)) ) + if user_ids: + connection.execute(delete(User).where(User.id.in_(user_ids))) connection.execute( delete(LegacyPolicyMapping).where( LegacyPolicyMapping.policy_id.in_(policy_ids) @@ -148,7 +171,7 @@ def _cleanup(engine, model_id) -> None: ) -def test_distinct_saved_rows_share_policy_and_preserve_nullable_names() -> None: +def test_saved_rows_share_policy_and_reuse_only_the_same_mapped_user() -> None: engine = create_engine(_disposable_url()) sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) model_id = None @@ -162,6 +185,12 @@ def test_distinct_saved_rows_share_policy_and_preserve_nullable_names() -> None: reform_id=102, reform_label=None, ) + third_saved = _saved( + legacy_id=203, + reform_id=101, + reform_label="Other user", + user_id="legacy-user-two", + ) with sessions.begin() as session: first = persist_legacy_user_policy( @@ -176,6 +205,12 @@ def test_distinct_saved_rows_share_policy_and_preserve_nullable_names() -> None: second_reform, source_revision=1, ) + third = persist_legacy_user_policy( + session, + third_saved, + first_reform, + source_revision=1, + ) with sessions.begin() as session: retry = persist_legacy_user_policy( session, @@ -186,6 +221,7 @@ def test_distinct_saved_rows_share_policy_and_preserve_nullable_names() -> None: assert first.policy_id == second.policy_id assert first.association_id != second.association_id + assert third.policy_id == first.policy_id assert retry.association_id == first.association_id assert retry.association_created is False with sessions() as session: @@ -195,9 +231,23 @@ def test_distinct_saved_rows_share_policy_and_preserve_nullable_names() -> None: assert {association.name for association in associations} == { "Reform", None, + "Other user", } assert all(association.description is None for association in associations) + association_by_id = { + association.id: association for association in associations + } + assert association_by_id[first.association_id].user_id == ( + association_by_id[second.association_id].user_id + ) + assert association_by_id[third.association_id].user_id != ( + association_by_id[first.association_id].user_id + ) assert session.scalar(select(func.count()).select_from(Policy)) == 1 + assert session.scalar(select(func.count()).select_from(User)) == 2 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 2 + ) assert ( session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) == 2 @@ -206,13 +256,54 @@ def test_distinct_saved_rows_share_policy_and_preserve_nullable_names() -> None: session.scalar( select(func.count()).select_from(LegacyUserPolicyMapping) ) - == 2 + == 3 ) finally: _cleanup(engine, model_id) engine.dispose() +def test_concurrent_first_use_resolves_one_legacy_user_mapping() -> None: + engine = create_engine(_disposable_url()) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + legacy_user_id = f"phase10-concurrent-{uuid4()}" + barrier = Barrier(2) + + def resolve() -> UUID: + with sessions.begin() as session: + barrier.wait() + return resolve_legacy_user_id( + session, + legacy_user_id=legacy_user_id, + primary_country="us", + ) + + try: + with ThreadPoolExecutor(max_workers=2) as executor: + user_ids = list(executor.map(lambda _index: resolve(), range(2))) + + assert user_ids[0] == user_ids[1] + with sessions() as session: + mappings = session.scalars( + select(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == legacy_user_id + ) + ).all() + assert len(mappings) == 1 + assert session.get(User, user_ids[0]) is not None + finally: + with sessions.begin() as session: + mapping = session.get(LegacyUserMapping, legacy_user_id) + if mapping is not None: + user_id = mapping.user_id + session.delete(mapping) + session.flush() + user = session.get(User, user_id) + if user is not None: + session.delete(user) + engine.dispose() + + def test_label_and_v1_only_updates_advance_the_complete_row_fingerprint() -> None: engine = create_engine(_disposable_url()) sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) @@ -311,6 +402,10 @@ def test_complete_transaction_rolls_back_and_native_delete_is_isolated() -> None with sessions() as session: assert session.scalar(select(func.count()).select_from(Policy)) == 0 assert session.scalar(select(func.count()).select_from(UserPolicy)) == 0 + assert session.scalar(select(func.count()).select_from(User)) == 0 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 0 + ) assert ( session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) == 0 @@ -342,6 +437,10 @@ def test_complete_transaction_rolls_back_and_native_delete_is_isolated() -> None == 0 ) assert session.get(Policy, created.policy_id) is not None + assert session.scalar(select(func.count()).select_from(User)) == 1 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 1 + ) assert ( session.scalar( select(func.count()) diff --git a/tests/unit/test_query_parameters.py b/tests/unit/test_query_parameters.py index 6313286b7..fecc2748b 100644 --- a/tests/unit/test_query_parameters.py +++ b/tests/unit/test_query_parameters.py @@ -109,12 +109,13 @@ def test_duplicate_scalar_is_rejected_and_explicit_list_is_preserved() -> None: def test_multidict_adapter_preserves_the_canonical_contract() -> None: policy_id = uuid4() + user_id = uuid4() parsed = parse_multidict_query( UserPolicyCollectionQuery, MultiDict( [ ("country_id", "UK"), - ("user_id", "auth0|example"), + ("user_id", str(user_id)), ("policy_id", str(policy_id)), ("offset", "2"), ] @@ -122,7 +123,7 @@ def test_multidict_adapter_preserves_the_canonical_contract() -> None: ) assert parsed.country_id == "uk" - assert parsed.user_id == "auth0|example" + assert parsed.user_id == user_id assert parsed.policy_id == policy_id assert parsed.offset == 2 diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 8ed46ac87..934fc1614 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -217,11 +217,12 @@ def test_v2_files_are_mechanically_separate_from_v1() -> None: assert all("migrations/v1" not in str(path) for path in v2_files) -def test_v2_revision_chain_has_generated_baseline_stage_9_and_phase_10() -> None: +def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> None: config = Config(str(REPO / "alembic-v2.ini")) script = ScriptDirectory.from_config(config) - assert script.get_heads() == ["c21c4a807a49"] + assert script.get_heads() == ["af34023a728f"] assert [revision.revision for revision in script.walk_revisions()] == [ + "af34023a728f", "c21c4a807a49", "711ec2f0a5a5", "68b4a5ae5dc5", @@ -250,8 +251,8 @@ def test_v2_revision_chain_has_generated_baseline_stage_9_and_phase_10() -> None assert "fk_regions_default_dataset_model_datasets" in baseline assert "uq_datasets_model_name" in baseline assert "ck_datasets_output_storage_path" in baseline - assert baseline.count("op.create_table(") == len(V2_TABLE_NAMES) - 2 - assert baseline.count("op.drop_table(") == len(V2_TABLE_NAMES) - 2 + assert baseline.count("op.create_table(") == len(V2_TABLE_NAMES) - 3 + assert baseline.count("op.drop_table(") == len(V2_TABLE_NAMES) - 3 corrected_enum_names = set( re.findall( @@ -343,6 +344,27 @@ def test_saved_policy_revision_tracking_was_generated_after_phase_10() -> None: assert "op.bulk_insert(" not in revision +def test_legacy_user_uuid_mapping_revision_is_generated_and_reversible() -> None: + revision = ( + REPO / "migrations/v2/versions/af34023a728f_map_legacy_users_to_v2_uuids.py" + ).read_text(encoding="utf-8") + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert 'down_revision: Union[str, None] = "c21c4a807a49"' in revision + assert revision.count("Post-generation correction:") == 2 + assert 'postgresql_using="user_id::uuid"' in revision + assert 'postgresql_using="user_id::text"' in revision + assert 'op.create_table(\n "legacy_user_mappings"' in revision + assert 'op.drop_table("legacy_user_mappings")' in revision + assert "fk_user_policies_user_id_users" in revision + assert "uq_legacy_user_mappings_user_id" in revision + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision + + def test_alembic_rejects_unknown_missing_and_divergent_history(tmp_path: Path) -> None: original = REPO / "migrations/v2" missing = tmp_path / "missing" diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index 6d27d6ecd..89f7923c3 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -67,7 +67,7 @@ def reject_ddl(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 34 +assert len(V2_METADATA.tables) == 35 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules """ diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index c8a7f376e..2a4cc854e 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -11,6 +11,7 @@ from policyengine_api.data.v2.models import ( Dynamic, LegacyPolicyMapping, + LegacyUserMapping, LegacyUserPolicyMapping, Parameter, ParameterValue, @@ -240,19 +241,20 @@ def test_policy_parameter_value_owner_period_and_identity_constraints() -> None: def test_user_policy_allows_duplicates_but_requires_policy_country() -> None: engine = _relational_sqlite_engine() _model, _version, _parameter, policy = _policy_graph(content_hash="c" * 64) + user = User(primary_country="us") with Session(engine) as session: - session.add(policy) + session.add_all([policy, user]) session.commit() first = UserPolicy( country_id="us", - user_id="auth0|caller", + user_id=user.id, policy_id=policy.id, name="First", ) second = UserPolicy( country_id="us", - user_id="auth0|caller", + user_id=user.id, policy_id=policy.id, name="Second", ) @@ -264,7 +266,7 @@ def test_user_policy_allows_duplicates_but_requires_policy_country() -> None: session.add( UserPolicy( country_id="uk", - user_id="auth0|caller", + user_id=user.id, policy_id=policy.id, ) ) @@ -318,11 +320,12 @@ def test_legacy_policy_mapping_allows_many_sources_for_one_policy() -> None: def test_legacy_user_policy_mapping_destination_is_unique_and_cascades() -> None: engine = _relational_sqlite_engine() _model, _version, _parameter, policy = _policy_graph(content_hash="e" * 64) + user = User(primary_country="us") with Session(engine) as session: association = UserPolicy( country_id="us", - user_id="legacy-user", + user=user, policy=policy, ) mapping = LegacyUserPolicyMapping( @@ -365,6 +368,38 @@ def test_legacy_user_policy_mapping_destination_is_unique_and_cascades() -> None engine.dispose() +def test_legacy_user_mapping_is_one_to_one_and_profile_fields_are_optional() -> None: + engine = _relational_sqlite_engine() + + with Session(engine) as session: + user = User(primary_country="us") + session.add(user) + session.commit() + mapping = LegacyUserMapping( + legacy_user_id="legacy-user", + user_id=user.id, + ) + session.add(mapping) + session.commit() + + assert mapping.user_id == user.id + assert user.first_name is None + assert user.last_name is None + assert user.email is None + assert mapping.created_at is not None + + session.add( + LegacyUserMapping( + legacy_user_id="another-legacy-user", + user_id=user.id, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + def test_v2_models_do_not_create_a_parallel_sqlalchemy_orm_layer() -> None: models_directory = ( Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" / "models" diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index ff2ba189b..f22449932 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -12,6 +12,7 @@ Household, HouseholdJob, LegacyPolicyMapping, + LegacyUserMapping, LegacyUserPolicyMapping, ParameterValue, Policy, @@ -54,6 +55,7 @@ def test_domain_models_are_grouped_into_topic_scoped_modules() -> None: UserHouseholdAssociation: "associations", UserPolicy: "associations", LegacyPolicyMapping: "policy_mappings", + LegacyUserMapping: "policy_mappings", LegacyUserPolicyMapping: "policy_mappings", UserReportAssociation: "associations", UserSimulationAssociation: "associations", @@ -124,6 +126,7 @@ def test_user_owned_associations_have_relational_integrity() -> None: "user_simulation_associations", "simulation_associations", ), + (UserPolicy, "user_policies", "policy_associations"), (UserReportAssociation, "user_report_associations", "report_associations"), ) @@ -190,8 +193,10 @@ def test_policy_parameter_values_use_jsonb_and_enforce_period_identity() -> None def test_user_policy_is_an_independent_country_scoped_association() -> None: associations = V2_METADATA.tables["user_policies"] - assert associations.c.user_id.type.length == 255 - assert list(associations.c.user_id.foreign_keys) == [] + assert isinstance(associations.c.user_id.type, sa.Uuid) + user_foreign_key = next(iter(associations.c.user_id.foreign_keys)) + assert user_foreign_key.target_fullname == "users.id" + assert user_foreign_key.ondelete == "CASCADE" assert {"country", "label"}.isdisjoint(associations.c.keys()) assert {"country_id", "name", "description"}.issubset(associations.c.keys()) assert associations.c.name.nullable @@ -217,6 +222,26 @@ def test_user_policy_is_an_independent_country_scoped_association() -> None: "policies.country_id", ] assert policy_country.ondelete == "RESTRICT" + assert sa.inspect(UserPolicy).relationships["user"].back_populates == ( + "policy_associations" + ) + + +def test_legacy_user_mapping_is_one_to_one() -> None: + mappings = V2_METADATA.tables["legacy_user_mappings"] + + assert mappings.primary_key.columns.keys() == ["legacy_user_id"] + assert mappings.c.legacy_user_id.type.length == 255 + assert mappings.c.created_at.type.timezone + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in mappings.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("user_id",) in unique_columns + user_foreign_key = next(iter(mappings.c.user_id.foreign_keys)) + assert user_foreign_key.target_fullname == "users.id" + assert user_foreign_key.ondelete == "RESTRICT" def test_legacy_policy_mapping_is_many_to_one_by_destination() -> None: @@ -315,6 +340,9 @@ def test_user_primary_country_is_required_and_limited_to_supported_values() -> N users = V2_METADATA.tables["users"] assert not users.c.primary_country.nullable + assert users.c.first_name.nullable + assert users.c.last_name.nullable + assert users.c.email.nullable assert users.c.primary_country.type.length == 2 assert "ck_users_primary_country" in { constraint.name for constraint in users.constraints diff --git a/tests/unit/v2/test_user_policy_legacy.py b/tests/unit/v2/test_user_policy_legacy.py index 9edbc816f..694fcd8eb 100644 --- a/tests/unit/v2/test_user_policy_legacy.py +++ b/tests/unit/v2/test_user_policy_legacy.py @@ -18,6 +18,7 @@ POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +USER_ID = UUID("00000000-0000-0000-0000-000000000070") def _snapshot(**changes) -> LegacyUserPolicySnapshot: @@ -97,7 +98,7 @@ def _apply_update( ): association = UserPolicy( country_id="us", - user_id="auth0|one", + user_id=USER_ID, policy_id=POLICY_ID, name="Native name", description="Native description", @@ -117,6 +118,7 @@ def _apply_update( mapping=mapping, snapshot=_snapshot(reform_label="Legacy rename", year="2027"), fingerprint=fingerprint, + user_id=USER_ID, policy_id=POLICY_ID, changed_fields=changed_fields, source_revision=source_revision, diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py index aca9699b8..70d6c9e21 100644 --- a/tests/unit/v2/test_user_policy_routes.py +++ b/tests/unit/v2/test_user_policy_routes.py @@ -15,6 +15,7 @@ from policyengine_api.data.v2.user_policies.persistence import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, + AssociationUserNotFoundError, ) from policyengine_api.data.v2.user_policies.query import ( UserPolicyNotFoundError, @@ -30,6 +31,7 @@ ASSOCIATION_ID = UUID("00000000-0000-0000-0000-000000000060") POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +USER_ID = UUID("00000000-0000-0000-0000-000000000070") CREATED_AT = datetime(2026, 1, 1, tzinfo=timezone.utc) @@ -37,7 +39,7 @@ def _association_read( *, association_id: UUID = ASSOCIATION_ID, country_id: str = "us", - user_id: str = "auth0|caller", + user_id: UUID = USER_ID, policy_id: UUID = POLICY_ID, name: str | None = "Saved reform", description: str | None = "Personal note", @@ -140,7 +142,7 @@ def fallback(resource: str): def _body(**changes) -> dict[str, object]: body: dict[str, object] = { "country_id": "us", - "user_id": "auth0|caller", + "user_id": str(USER_ID), "policy_id": str(POLICY_ID), "name": "Saved reform", "description": "Personal note", @@ -163,7 +165,7 @@ def test_create_returns_complete_distinct_association_contract() -> None: "item": { "id": str(ASSOCIATION_ID), "country_id": "us", - "user_id": "auth0|caller", + "user_id": str(USER_ID), "policy_id": str(POLICY_ID), "name": "Saved reform", "description": "Personal note", @@ -173,7 +175,7 @@ def test_create_returns_complete_distinct_association_contract() -> None: }, } command = service.calls[0][1] - assert command.user_id == "auth0|caller" + assert command.user_id == USER_ID assert command.policy_id == POLICY_ID assert flask_calls["count"] == 0 @@ -194,14 +196,14 @@ def test_repeated_create_calls_service_twice_without_link_deduplication() -> Non ] -def test_create_rejects_country_mismatch_and_bounded_fields() -> None: +def test_create_rejects_country_mismatch_and_invalid_fields() -> None: service = FakeUserPolicyService() client, _flask_calls = _client(service) mismatch = client.post("/v2/user-policies?country_id=uk", json=_body()) - empty_user = client.post( + invalid_user = client.post( "/v2/user-policies?country_id=us", - json=_body(user_id=" "), + json=_body(user_id="not-a-uuid"), ) long_name = client.post( "/v2/user-policies?country_id=us", @@ -209,7 +211,7 @@ def test_create_rejects_country_mismatch_and_bounded_fields() -> None: ) assert mismatch.status_code == 400 - assert empty_user.status_code == 422 + assert invalid_user.status_code == 422 assert long_name.status_code == 422 assert service.calls == [] @@ -220,7 +222,7 @@ def test_detail_and_list_use_country_user_policy_and_pagination_filters() -> Non detail = client.get(f"/v2/user-policies/{ASSOCIATION_ID}?country_id=US") page = client.get( - "/v2/user-policies?country_id=us&user_id=auth0%7Ccaller" + f"/v2/user-policies?country_id=us&user_id={USER_ID}" f"&policy_id={POLICY_ID}&offset=2&limit=3" ) @@ -235,7 +237,7 @@ def test_detail_and_list_use_country_user_policy_and_pagination_filters() -> Non "list_user_policies", { "country_id": "us", - "user_id": "auth0|caller", + "user_id": USER_ID, "policy_id": POLICY_ID, "offset": 2, "limit": 3, @@ -256,9 +258,9 @@ def test_collection_rejects_missing_unknown_duplicate_and_invalid_queries() -> N responses = [ client.get("/v2/user-policies?country_id=us"), - client.get("/v2/user-policies?country_id=us&user_id=u&search=reform"), - client.get("/v2/user-policies?country_id=us&country_id=uk&user_id=u"), - client.get("/v2/user-policies?country_id=us&user_id=u&limit=501"), + client.get(f"/v2/user-policies?country_id=us&user_id={USER_ID}&search=reform"), + client.get(f"/v2/user-policies?country_id=us&country_id=uk&user_id={USER_ID}"), + client.get(f"/v2/user-policies?country_id=us&user_id={USER_ID}&limit=501"), ] assert [response.status_code for response in responses] == [422] * 4 @@ -313,6 +315,7 @@ def test_delete_returns_no_content_and_uses_country_scoped_identity() -> None: [ (AssociationCountryConflictError("different country"), 400, "country"), (AssociationPolicyNotFoundError("policy was not found"), 404, "not found"), + (AssociationUserNotFoundError("user was not found"), 404, "not found"), (UserPolicyNotFoundError("association was not found"), 404, "not found"), (V2ConfigurationError("postgresql://secret"), 503, "unavailable"), ( @@ -358,7 +361,8 @@ def test_openapi_publishes_complete_no_auth_association_contracts() -> None: parameters = {item["name"]: item for item in collection["parameters"]} assert parameters["country_id"]["required"] is True assert parameters["user_id"]["required"] is True - assert "Unverified caller-supplied" in parameters["user_id"]["description"] + assert "does not prove caller control" in parameters["user_id"]["description"] + assert parameters["user_id"]["schema"]["format"] == "uuid" assert parameters["limit"]["schema"]["maximum"] == 500 assert set(schema["paths"]["/v2/user-policies"]) == {"get", "post"} assert set(schema["paths"]["/v2/user-policies/{association_id}"]) == { diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py index 7cd422eb4..722605fe4 100644 --- a/tests/unit/v2/test_user_policy_service.py +++ b/tests/unit/v2/test_user_policy_service.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import datetime, timezone +from uuid import UUID import pytest import sqlalchemy as sa @@ -16,12 +17,14 @@ Policy, TaxBenefitModel, TaxBenefitModelVersion, + User, UserPolicy, V2_METADATA, ) from policyengine_api.data.v2.user_policies.persistence import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, + AssociationUserNotFoundError, ) from policyengine_api.data.v2.user_policies.query import UserPolicyNotFoundError from policyengine_api.data.v2.user_policies.schemas import ( @@ -31,6 +34,10 @@ from policyengine_api.data.v2.user_policies.service import V2UserPolicyService +USER_ID = UUID("00000000-0000-0000-0000-000000000070") +OTHER_USER_ID = UUID("00000000-0000-0000-0000-000000000071") + + @pytest.fixture def association_store(): engine = create_engine("sqlite://") @@ -68,7 +75,13 @@ def enable_foreign_keys(dbapi_connection, _connection_record) -> None: value_json=0.2, start_date=datetime(2026, 1, 1, tzinfo=timezone.utc), ) - session.add(value) + session.add_all( + [ + value, + User(id=USER_ID, primary_country="us"), + User(id=OTHER_USER_ID, primary_country="us"), + ] + ) session.flush() identity = (policy.id, value.id) @@ -79,7 +92,7 @@ def enable_foreign_keys(dbapi_connection, _connection_record) -> None: def _command(policy_id, **changes) -> UserPolicyCreateCommand: values = { "country_id": "us", - "user_id": "auth0|caller", + "user_id": USER_ID, "policy_id": policy_id, "name": "Saved reform", "description": "Personal note", @@ -88,7 +101,7 @@ def _command(policy_id, **changes) -> UserPolicyCreateCommand: return UserPolicyCreateCommand.model_validate(values) -def test_create_allows_distinct_duplicate_links_and_unverified_user( +def test_create_allows_distinct_duplicate_links_for_an_existing_user( association_store, ) -> None: service, sessions, (policy_id, _value_id) = association_store @@ -97,7 +110,7 @@ def test_create_allows_distinct_duplicate_links_and_unverified_user( second = service.create_user_policy(_command(policy_id, name="Second save")) assert first.id != second.id - assert first.user_id == "auth0|caller" + assert first.user_id == USER_ID assert second.policy_id == policy_id with sessions() as session: assert len(session.exec(select(UserPolicy)).all()) == 2 @@ -110,6 +123,13 @@ def test_create_rejects_missing_policy_and_country_conflict( with pytest.raises(AssociationPolicyNotFoundError): service.create_user_policy(_command("00000000-0000-0000-0000-000000000099")) + with pytest.raises(AssociationUserNotFoundError): + service.create_user_policy( + _command( + policy_id, + user_id="00000000-0000-0000-0000-000000000099", + ) + ) with pytest.raises(AssociationCountryConflictError): service.create_user_policy(_command(policy_id, country_id="uk")) @@ -120,9 +140,7 @@ def test_detail_list_filter_and_pagination_are_country_scoped( service, _sessions, (policy_id, _value_id) = association_store first = service.create_user_policy(_command(policy_id, name="First")) service.create_user_policy(_command(policy_id, name="Second")) - service.create_user_policy( - _command(policy_id, user_id="another-user", name="Other") - ) + service.create_user_policy(_command(policy_id, user_id=OTHER_USER_ID, name="Other")) detail = service.get_user_policy( country_id="us", @@ -130,13 +148,13 @@ def test_detail_list_filter_and_pagination_are_country_scoped( ) page = service.list_user_policies( country_id="us", - user_id="auth0|caller", + user_id=USER_ID, policy_id=policy_id, limit=1, ) second_page = service.list_user_policies( country_id="us", - user_id="auth0|caller", + user_id=USER_ID, offset=1, limit=1, ) @@ -173,7 +191,7 @@ def test_patch_changes_only_supplied_fields_and_supports_null_clearing( assert cleared.updated_at >= created.updated_at assert (cleared.country_id, cleared.user_id, cleared.policy_id) == ( "us", - "auth0|caller", + USER_ID, policy_id, ) From 7f771e63429e5907cffeb4e664b35bbf57603b05 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:22:49 +0400 Subject: [PATCH 04/18] Restore user relationship ordering --- policyengine_api/data/v2/models/users.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/policyengine_api/data/v2/models/users.py b/policyengine_api/data/v2/models/users.py index ca85c46ce..87db70d6b 100644 --- a/policyengine_api/data/v2/models/users.py +++ b/policyengine_api/data/v2/models/users.py @@ -37,15 +37,15 @@ class User(IdentifiedModel, table=True): back_populates="user", cascade_delete=True, ) - simulation_associations: list["UserSimulationAssociation"] = Relationship( + policy_associations: list["UserPolicy"] = Relationship( back_populates="user", cascade_delete=True, ) - report_associations: list["UserReportAssociation"] = Relationship( + simulation_associations: list["UserSimulationAssociation"] = Relationship( back_populates="user", cascade_delete=True, ) - policy_associations: list["UserPolicy"] = Relationship( + report_associations: list["UserReportAssociation"] = Relationship( back_populates="user", cascade_delete=True, ) From e16a7db88740cad56c115a9e13c3450b35a75da2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:52:09 +0400 Subject: [PATCH 05/18] Organize v2 routes and services by layer --- policyengine_api/asgi_factory.py | 10 +++++----- policyengine_api/fastapi_routes/dependencies.py | 8 ++++---- policyengine_api/fastapi_routes/v2/__init__.py | 1 + .../fastapi_routes/v2/metadata/__init__.py | 1 + .../metadata/common.py} | 4 ++-- .../metadata/geography_routes.py} | 4 ++-- .../metadata/model_routes.py} | 4 ++-- .../metadata/parameter_routes.py} | 4 ++-- .../{v2_policies.py => v2/policy_routes.py} | 0 .../{v2_metadata.py => v2/routes.py} | 14 +++++++------- .../user_policy_routes.py} | 0 policyengine_api/services/policy_mirroring.py | 2 +- .../services/user_policy_mirroring.py | 2 +- policyengine_api/services/v2/__init__.py | 1 + .../query.py => services/v2/metadata_service.py} | 6 +++--- .../service.py => services/v2/policy_service.py} | 0 .../v2/user_policy_service.py} | 0 tests/integration/test_v1_policy_dual_write.py | 2 +- .../integration/test_v1_user_policy_dual_write.py | 2 +- tests/integration/test_v2_metadata_routes.py | 4 ++-- tests/unit/v2/test_metadata_routes.py | 6 +++--- ...metadata_query.py => test_metadata_service.py} | 15 +++++++-------- tests/unit/v2/test_policy_routes.py | 2 +- tests/unit/v2/test_user_policy_service.py | 2 +- 24 files changed, 48 insertions(+), 46 deletions(-) create mode 100644 policyengine_api/fastapi_routes/v2/__init__.py create mode 100644 policyengine_api/fastapi_routes/v2/metadata/__init__.py rename policyengine_api/fastapi_routes/{v2_metadata_common.py => v2/metadata/common.py} (96%) rename policyengine_api/fastapi_routes/{v2_metadata_geography.py => v2/metadata/geography_routes.py} (97%) rename policyengine_api/fastapi_routes/{v2_metadata_models.py => v2/metadata/model_routes.py} (97%) rename policyengine_api/fastapi_routes/{v2_metadata_parameters.py => v2/metadata/parameter_routes.py} (97%) rename policyengine_api/fastapi_routes/{v2_policies.py => v2/policy_routes.py} (100%) rename policyengine_api/fastapi_routes/{v2_metadata.py => v2/routes.py} (86%) rename policyengine_api/fastapi_routes/{v2_user_policies.py => v2/user_policy_routes.py} (100%) create mode 100644 policyengine_api/services/v2/__init__.py rename policyengine_api/{data/v2/catalog/query.py => services/v2/metadata_service.py} (92%) rename policyengine_api/{data/v2/policies/service.py => services/v2/policy_service.py} (100%) rename policyengine_api/{data/v2/user_policies/service.py => services/v2/user_policy_service.py} (100%) rename tests/unit/v2/{test_metadata_query.py => test_metadata_service.py} (98%) diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index 933d0a62b..e57bd3e28 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -19,12 +19,12 @@ from policyengine_api.fastapi_routes.specification import ( build_specification_router, ) -from policyengine_api.fastapi_routes.v2_policies import ( +from policyengine_api.fastapi_routes.v2.policy_routes import ( PolicyRequestTooLargeError, policy_error_response, ) -from policyengine_api.fastapi_routes.v2_metadata import build_v2_metadata_router -from policyengine_api.fastapi_routes.v2_user_policies import ( +from policyengine_api.fastapi_routes.v2.routes import build_v2_router +from policyengine_api.fastapi_routes.v2.user_policy_routes import ( user_policy_error_response, ) from policyengine_api.migration_flags import ( @@ -127,7 +127,7 @@ async def typed_v2_request_validation_error( ) if request.url.path.startswith("/v2/policies"): return policy_error_response(422, "Invalid v2 policy request") - from policyengine_api.fastapi_routes.v2_metadata_common import ( + from policyengine_api.fastapi_routes.v2.metadata.common import ( error_response, ) @@ -181,7 +181,7 @@ def log_native_route(status_code: int) -> None: _asgi_request_id.reset(context_token) app.include_router(build_core_health_router(dependencies)) - app.include_router(build_v2_metadata_router(dependencies)) + app.include_router(build_v2_router(dependencies)) if route_settings.health is RouteImplementation.FASTAPI_NATIVE: app.include_router(build_readiness_router(dependencies)) if route_settings.specification is RouteImplementation.FASTAPI_NATIVE: diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 3b7243b01..f8fab68fa 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -83,10 +83,10 @@ def _running_policyengine_version() -> str: def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: - from policyengine_api.data.v2.catalog.query import V2MetadataQueryService from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.services.v2.metadata_service import V2MetadataService - return V2MetadataQueryService( + return V2MetadataService( get_v2_session_factory()(), running_policyengine_version=_running_policyengine_version(), ) @@ -94,7 +94,7 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: def _default_v2_policy_service_factory() -> V2PolicyResourceService: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.data.v2.policies.service import V2PolicyService + from policyengine_api.services.v2.policy_service import V2PolicyService return V2PolicyService( get_v2_session_factory(), @@ -104,7 +104,7 @@ def _default_v2_policy_service_factory() -> V2PolicyResourceService: def _default_v2_user_policy_service_factory() -> V2UserPolicyResourceService: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.data.v2.user_policies.service import V2UserPolicyService + from policyengine_api.services.v2.user_policy_service import V2UserPolicyService return V2UserPolicyService(get_v2_session_factory()) diff --git a/policyengine_api/fastapi_routes/v2/__init__.py b/policyengine_api/fastapi_routes/v2/__init__.py new file mode 100644 index 000000000..c4f3cad04 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/__init__.py @@ -0,0 +1 @@ +"""Native API v2 HTTP route modules.""" diff --git a/policyengine_api/fastapi_routes/v2/metadata/__init__.py b/policyengine_api/fastapi_routes/v2/metadata/__init__.py new file mode 100644 index 000000000..7cff9ae11 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/__init__.py @@ -0,0 +1 @@ +"""Native API v2 metadata HTTP route modules.""" diff --git a/policyengine_api/fastapi_routes/v2_metadata_common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py similarity index 96% rename from policyengine_api/fastapi_routes/v2_metadata_common.py rename to policyengine_api/fastapi_routes/v2/metadata/common.py index de54e75c4..5b5c447ae 100644 --- a/policyengine_api/fastapi_routes/v2_metadata_common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -1,4 +1,4 @@ -"""Shared response handling for dormant v2 metadata resources.""" +"""Shared response handling for API v2 metadata routes.""" from __future__ import annotations @@ -8,7 +8,7 @@ from pydantic import BaseModel from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.query import ( +from policyengine_api.services.v2.metadata_service import ( InvalidMetadataPageError, InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, diff --git a/policyengine_api/fastapi_routes/v2_metadata_geography.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py similarity index 97% rename from policyengine_api/fastapi_routes/v2_metadata_geography.py rename to policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index fc366b0f6..82d8539cd 100644 --- a/policyengine_api/fastapi_routes/v2_metadata_geography.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -1,4 +1,4 @@ -"""Dataset, region, and economy-option preview routes.""" +"""API v2 dataset, region, and economy-option routes.""" from __future__ import annotations @@ -17,7 +17,7 @@ MetadataRegionType, ) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -from policyengine_api.fastapi_routes.v2_metadata_common import ( +from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, read_resource, ) diff --git a/policyengine_api/fastapi_routes/v2_metadata_models.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py similarity index 97% rename from policyengine_api/fastapi_routes/v2_metadata_models.py rename to policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 8a1147f62..258c3edfa 100644 --- a/policyengine_api/fastapi_routes/v2_metadata_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -1,4 +1,4 @@ -"""Tax-benefit model, version, and variable preview routes.""" +"""API v2 tax-benefit model, version, and variable routes.""" from __future__ import annotations @@ -18,7 +18,7 @@ MetadataVariablePageResponse, ) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -from policyengine_api.fastapi_routes.v2_metadata_common import ( +from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, read_resource, ) diff --git a/policyengine_api/fastapi_routes/v2_metadata_parameters.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py similarity index 97% rename from policyengine_api/fastapi_routes/v2_metadata_parameters.py rename to policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index b4f17cb20..842a1e5fb 100644 --- a/policyengine_api/fastapi_routes/v2_metadata_parameters.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -1,4 +1,4 @@ -"""Parameter and canonical parameter-value preview routes.""" +"""API v2 parameter and canonical parameter-value routes.""" from __future__ import annotations @@ -16,7 +16,7 @@ MetadataParameterValuePageResponse, ) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -from policyengine_api.fastapi_routes.v2_metadata_common import ( +from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, read_resource, ) diff --git a/policyengine_api/fastapi_routes/v2_policies.py b/policyengine_api/fastapi_routes/v2/policy_routes.py similarity index 100% rename from policyengine_api/fastapi_routes/v2_policies.py rename to policyengine_api/fastapi_routes/v2/policy_routes.py diff --git a/policyengine_api/fastapi_routes/v2_metadata.py b/policyengine_api/fastapi_routes/v2/routes.py similarity index 86% rename from policyengine_api/fastapi_routes/v2_metadata.py rename to policyengine_api/fastapi_routes/v2/routes.py index b5a61fec4..235287898 100644 --- a/policyengine_api/fastapi_routes/v2_metadata.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -1,4 +1,4 @@ -"""Dormant, read-only API v2 metadata resource routes.""" +"""Compose the native API v2 policy and metadata routes.""" from __future__ import annotations @@ -7,22 +7,22 @@ from policyengine_api.data.v2.catalog.schemas import MetadataErrorResponse from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -from policyengine_api.fastapi_routes.v2_metadata_geography import ( +from policyengine_api.fastapi_routes.v2.metadata.geography_routes import ( build_v2_metadata_geography_router, ) -from policyengine_api.fastapi_routes.v2_metadata_models import ( +from policyengine_api.fastapi_routes.v2.metadata.model_routes import ( build_v2_metadata_model_router, ) -from policyengine_api.fastapi_routes.v2_metadata_parameters import ( +from policyengine_api.fastapi_routes.v2.metadata.parameter_routes import ( build_v2_metadata_parameter_router, ) -from policyengine_api.fastapi_routes.v2_policies import build_v2_policy_router -from policyengine_api.fastapi_routes.v2_user_policies import ( +from policyengine_api.fastapi_routes.v2.policy_routes import build_v2_policy_router +from policyengine_api.fastapi_routes.v2.user_policy_routes import ( build_v2_user_policy_router, ) -def build_v2_metadata_router( +def build_v2_router( dependencies: NativeRouteDependencies, ) -> APIRouter: """Build isolated resource routes without loading v2 configuration.""" diff --git a/policyengine_api/fastapi_routes/v2_user_policies.py b/policyengine_api/fastapi_routes/v2/user_policy_routes.py similarity index 100% rename from policyengine_api/fastapi_routes/v2_user_policies.py rename to policyengine_api/fastapi_routes/v2/user_policy_routes.py diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py index aa1a3449f..d6313537c 100644 --- a/policyengine_api/services/policy_mirroring.py +++ b/policyengine_api/services/policy_mirroring.py @@ -42,7 +42,7 @@ class PolicyMirrorUnavailableError(RuntimeError): def _default_mirror_factory() -> LegacyPolicyMirror: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.data.v2.policies.service import V2PolicyService + from policyengine_api.services.v2.policy_service import V2PolicyService return V2PolicyService(get_v2_session_factory()) diff --git a/policyengine_api/services/user_policy_mirroring.py b/policyengine_api/services/user_policy_mirroring.py index 5da708794..5f63a796a 100644 --- a/policyengine_api/services/user_policy_mirroring.py +++ b/policyengine_api/services/user_policy_mirroring.py @@ -53,7 +53,7 @@ class UserPolicyMirrorUnavailableError(RuntimeError): def _default_mirror_factory() -> LegacyUserPolicyMirror: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.data.v2.user_policies.service import V2UserPolicyService + from policyengine_api.services.v2.user_policy_service import V2UserPolicyService return V2UserPolicyService(get_v2_session_factory()) diff --git a/policyengine_api/services/v2/__init__.py b/policyengine_api/services/v2/__init__.py new file mode 100644 index 000000000..7452ae05a --- /dev/null +++ b/policyengine_api/services/v2/__init__.py @@ -0,0 +1 @@ +"""Application services for native API v2 resources.""" diff --git a/policyengine_api/data/v2/catalog/query.py b/policyengine_api/services/v2/metadata_service.py similarity index 92% rename from policyengine_api/data/v2/catalog/query.py rename to policyengine_api/services/v2/metadata_service.py index 8e24ffa60..93b894b7a 100644 --- a/policyengine_api/data/v2/catalog/query.py +++ b/policyengine_api/services/v2/metadata_service.py @@ -1,4 +1,4 @@ -"""Public read-only v2 metadata query service.""" +"""Session-owning application service for API v2 metadata reads.""" from __future__ import annotations @@ -28,13 +28,13 @@ "MetadataCatalogVersionNotFoundError", "MetadataResourceNotFoundError", "UnsupportedPreviewCountryError", - "V2MetadataQueryService", + "V2MetadataService", "validate_metadata_page", "validate_policyengine_version", ] -class V2MetadataQueryService( +class V2MetadataService( ModelQueryMethods, VariableQueryMethods, ParameterQueryMethods, diff --git a/policyengine_api/data/v2/policies/service.py b/policyengine_api/services/v2/policy_service.py similarity index 100% rename from policyengine_api/data/v2/policies/service.py rename to policyengine_api/services/v2/policy_service.py diff --git a/policyengine_api/data/v2/user_policies/service.py b/policyengine_api/services/v2/user_policy_service.py similarity index 100% rename from policyengine_api/data/v2/user_policies/service.py rename to policyengine_api/services/v2/user_policy_service.py diff --git a/tests/integration/test_v1_policy_dual_write.py b/tests/integration/test_v1_policy_dual_write.py index 6d16a5ba6..2469f9482 100644 --- a/tests/integration/test_v1_policy_dual_write.py +++ b/tests/integration/test_v1_policy_dual_write.py @@ -25,7 +25,7 @@ TaxBenefitModelVersion, ) from policyengine_api.data.v2.policies.legacy import persist_legacy_policy -from policyengine_api.data.v2.policies.service import V2PolicyService +from policyengine_api.services.v2.policy_service import V2PolicyService from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from policyengine_api.services.policy_mirroring import ( PolicyMirrorUnavailableError, diff --git a/tests/integration/test_v1_user_policy_dual_write.py b/tests/integration/test_v1_user_policy_dual_write.py index 643623e2d..5a25cc802 100644 --- a/tests/integration/test_v1_user_policy_dual_write.py +++ b/tests/integration/test_v1_user_policy_dual_write.py @@ -28,7 +28,7 @@ UserPolicy, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL -from policyengine_api.data.v2.user_policies.service import V2UserPolicyService +from policyengine_api.services.v2.user_policy_service import V2UserPolicyService from policyengine_api.services.policy_service import PolicyService from policyengine_api.services.user_policy_mirroring import ( UserPolicyMirrorUnavailableError, diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index a7cc3d79d..765bc6269 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -18,7 +18,7 @@ from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.data.v2.catalog.publication import publish_catalog -from policyengine_api.data.v2.catalog.query import V2MetadataQueryService +from policyengine_api.services.v2.metadata_service import V2MetadataService from policyengine_api.data.v2.models import ( Dataset, TaxBenefitModel, @@ -78,7 +78,7 @@ def v1_metadata(country_id: str): gateway_client_factory=lambda: None, metadata_reader_factory=lambda: None, specification_provider=lambda: {}, - v2_metadata_reader_factory=lambda: V2MetadataQueryService( + v2_metadata_reader_factory=lambda: V2MetadataService( Session(engine), running_policyengine_version=POLICYENGINE_VERSION, ), diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 457c22534..5e4a881e8 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -11,7 +11,7 @@ import pytest from policyengine_api.asgi_factory import create_asgi_app -from policyengine_api.data.v2.catalog.query import ( +from policyengine_api.services.v2.metadata_service import ( InvalidMetadataPageError, InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, @@ -435,7 +435,7 @@ def test_default_reader_uses_the_installed_policyengine_version( monkeypatch: pytest.MonkeyPatch, ) -> None: from policyengine_api.data.v2 import database - from policyengine_api.data.v2.catalog import query + from policyengine_api.services.v2 import metadata_service session = object() captured = {} @@ -447,7 +447,7 @@ def query_service(candidate_session, *, running_policyengine_version): return reader monkeypatch.setattr(database, "get_v2_session_factory", lambda: lambda: session) - monkeypatch.setattr(query, "V2MetadataQueryService", query_service) + monkeypatch.setattr(metadata_service, "V2MetadataService", query_service) monkeypatch.setattr( route_dependencies.importlib_metadata, "version", diff --git a/tests/unit/v2/test_metadata_query.py b/tests/unit/v2/test_metadata_service.py similarity index 98% rename from tests/unit/v2/test_metadata_query.py rename to tests/unit/v2/test_metadata_service.py index 2b57275eb..49010b17a 100644 --- a/tests/unit/v2/test_metadata_query.py +++ b/tests/unit/v2/test_metadata_service.py @@ -1,4 +1,4 @@ -"""Read-only query coverage for v2 metadata resources.""" +"""Read-only service coverage for v2 metadata resources.""" from __future__ import annotations @@ -12,14 +12,14 @@ from sqlalchemy.pool import StaticPool from sqlmodel import Session, create_engine, select -from policyengine_api.data.v2.catalog.query import ( +from policyengine_api.services.v2.metadata_service import ( InvalidMetadataPageError, InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, MetadataResourceNotFoundError, UnsupportedPreviewCountryError, - V2MetadataQueryService, + V2MetadataService, ) from policyengine_api.data.v2.models import ( Dataset, @@ -201,8 +201,8 @@ def _us_model_version(session: Session) -> TaxBenefitModelVersion: ).one() -def _service(session: Session) -> V2MetadataQueryService: - return V2MetadataQueryService( +def _service(session: Session) -> V2MetadataService: + return V2MetadataService( session, running_policyengine_version=POLICYENGINE_VERSION, ) @@ -485,7 +485,7 @@ def test_version_selection_rejects_invalid_absent_and_unsupported_requests( with pytest.raises(UnsupportedPreviewCountryError): service.list_variables("ca") with pytest.raises(MetadataCatalogUnavailableError): - V2MetadataQueryService( + V2MetadataService( catalog_session, running_policyengine_version="4.99.0", ).list_variables("us") @@ -561,7 +561,6 @@ def test_query_modules_import_no_policyengine_or_v1_metadata_source() -> None: "model_query.py", "parameter_query.py", "parameter_tree_query.py", - "query.py", "query_support.py", "region_query.py", "variable_query.py", @@ -619,5 +618,5 @@ def test_resource_service_methods_are_defined_in_their_query_modules() -> None: } for method_name, module_name in expected_modules.items(): - method = getattr(V2MetadataQueryService, method_name) + method = getattr(V2MetadataService, method_name) assert method.__module__.endswith(f".{module_name}") diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py index dc711c5a3..aa931739c 100644 --- a/tests/unit/v2/test_policy_routes.py +++ b/tests/unit/v2/test_policy_routes.py @@ -26,7 +26,7 @@ PolicyParameterValueRead, PolicyRead, ) -from policyengine_api.data.v2.policies.service import NativePolicyCreation +from policyengine_api.services.v2.policy_service import NativePolicyCreation from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies from policyengine_api.migration_flags import ( diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py index 722605fe4..f4ea8f543 100644 --- a/tests/unit/v2/test_user_policy_service.py +++ b/tests/unit/v2/test_user_policy_service.py @@ -31,7 +31,7 @@ UserPolicyCreateCommand, UserPolicyPatchCommand, ) -from policyengine_api.data.v2.user_policies.service import V2UserPolicyService +from policyengine_api.services.v2.user_policy_service import V2UserPolicyService USER_ID = UUID("00000000-0000-0000-0000-000000000070") From d761a30cf9496495c26692105b08c11b226f4820 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:17:43 +0400 Subject: [PATCH 06/18] Add static typing enforcement for API v2 --- .github/workflows/pr.yml | 16 ++ Makefile | 3 + docs/engineering/skills/testing.md | 8 + .../data/v2/policies/api_schemas.py | 4 +- policyengine_api/data/v2/policies/catalog.py | 4 +- policyengine_api/data/v2/policies/legacy.py | 27 +- .../data/v2/policies/persistence.py | 34 +-- policyengine_api/data/v2/policies/query.py | 16 +- .../data/v2/user_policies/api_schemas.py | 4 +- .../data/v2/user_policies/legacy.py | 8 +- .../data/v2/user_policies/query.py | 4 +- .../fastapi_routes/dependencies.py | 239 +++++++++++++++++- .../fastapi_routes/query_parameters.py | 4 +- .../fastapi_routes/v2/metadata/common.py | 11 +- .../fastapi_routes/v2/policy_routes.py | 8 +- .../fastapi_routes/v2/user_policy_routes.py | 8 +- pyproject.toml | 19 ++ uv.lock | 177 ++++++++++++- 18 files changed, 522 insertions(+), 72 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ffc5d0e0b..660617471 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -38,6 +38,22 @@ jobs: run: pip install ruff>=0.9.0 - name: Format check with ruff run: ruff format --check . + typecheck-v2: + name: Type-check API v2 + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup uv + uses: astral-sh/setup-uv@v6 + - name: Install locked dependencies + run: uv sync --frozen --extra dev + - name: Type-check API v2 + run: uv run --frozen --extra dev mypy quality-guards: name: Quality guards runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 41d451e53..8a97c7bb6 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,9 @@ test: quality-guards: python scripts/run_quality_guards.py +typecheck-v2: + uv run --frozen --extra dev mypy + debug-test: MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 pytest -vv --durations=0 tests diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index a8fb7e8f6..dca27a278 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -158,6 +158,14 @@ that result in the handoff instead of hiding it. ## Phase 10 Policy Migration +Run the configured static type check for the Phase 10 v2 query, route, +application-service, policy-persistence, and association-persistence modules. +The configured file set deliberately excludes the existing v1 implementation: + +```bash +uv run --frozen --extra dev mypy +``` + Run the shared query, SQLModel, native policy and association, v1 compatibility, configuration, readiness, and observability tests together: diff --git a/policyengine_api/data/v2/policies/api_schemas.py b/policyengine_api/data/v2/policies/api_schemas.py index bbe5d64df..27b70e7fd 100644 --- a/policyengine_api/data/v2/policies/api_schemas.py +++ b/policyengine_api/data/v2/policies/api_schemas.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime -from typing import Annotated, Generic, Literal, TypeVar +from typing import Annotated, Any, Generic, Literal, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, JsonValue, StringConstraints @@ -93,7 +93,7 @@ class PolicyErrorResponse(StrictPolicyAPIModel): message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] -POLICY_ERROR_RESPONSES = { +POLICY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": PolicyErrorResponse, "description": "The policy content or country selection is invalid.", diff --git a/policyengine_api/data/v2/policies/catalog.py b/policyengine_api/data/v2/policies/catalog.py index d9e23741f..c73bfcf64 100644 --- a/policyengine_api/data/v2/policies/catalog.py +++ b/policyengine_api/data/v2/policies/catalog.py @@ -4,7 +4,7 @@ from uuid import UUID -from sqlmodel import Session, select +from sqlmodel import Session, col, select from policyengine_api.constants import POLICYENGINE_VERSION from policyengine_api.data.v2.catalog.catalog_selection import select_catalog @@ -31,7 +31,7 @@ def _version_parameter_ids( session.exec( select(Parameter.id).where( Parameter.tax_benefit_model_version_id == model_version_id, - Parameter.id.in_(requested_ids), + col(Parameter.id).in_(requested_ids), ) ).all() ) diff --git a/policyengine_api/data/v2/policies/legacy.py b/policyengine_api/data/v2/policies/legacy.py index 053aac23f..e15dc9a58 100644 --- a/policyengine_api/data/v2/policies/legacy.py +++ b/policyengine_api/data/v2/policies/legacy.py @@ -8,10 +8,12 @@ from typing import Annotated from uuid import UUID -from policyengine_core.periods import period as parse_policyengine_period +from policyengine_core.periods import ( # type: ignore[import-untyped] + period as parse_policyengine_period, +) from pydantic import Field, field_validator from sqlalchemy.dialects.postgresql import insert -from sqlmodel import Session, select +from sqlmodel import Session, col, select from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION from policyengine_api.data.v2.catalog.catalog_selection import select_catalog @@ -20,6 +22,7 @@ from policyengine_api.data.v2.policies.persistence import persist_resolved_policy from policyengine_api.data.v2.policies.schemas import ( PolicyCreateCommand, + PolicyParameterValueCommand, ResolvedPolicyCreateCommand, StrictJsonValue, StrictPolicyCommand, @@ -103,7 +106,7 @@ def parse_legacy_period(value: str) -> tuple[datetime, datetime]: def _parameters_by_name( session: Session, *, - model_version_id, + model_version_id: UUID, names: set[str], ) -> dict[str, Parameter]: if not names: @@ -111,7 +114,7 @@ def _parameters_by_name( parameters = session.exec( select(Parameter).where( Parameter.tax_benefit_model_version_id == model_version_id, - Parameter.name.in_(names), + col(Parameter.name).in_(names), ) ).all() return {parameter.name: parameter for parameter in parameters} @@ -149,7 +152,7 @@ def translate_legacy_policy( "every legacy parameter path must exist in the running catalog" ) - parameter_values: list[dict[str, object]] = [] + parameter_values: list[PolicyParameterValueCommand] = [] for parameter_name in sorted(parameter_names): period_values = policy_json[parameter_name] if type(period_values) is not dict: @@ -163,12 +166,12 @@ def translate_legacy_policy( ) start_date, end_date = parse_legacy_period(period_name) parameter_values.append( - { - "parameter_id": parameters[parameter_name].id, - "value": value, - "start_date": start_date, - "end_date": end_date, - } + PolicyParameterValueCommand( + parameter_id=parameters[parameter_name].id, + value=value, + start_date=start_date, + end_date=end_date, + ) ) try: @@ -260,7 +263,7 @@ def persist_legacy_policy( source_policy_hash=snapshot.source_policy_hash, ) .on_conflict_do_nothing(constraint="uq_legacy_policy_mappings_country_legacy") - .returning(LegacyPolicyMapping.id) + .returning(col(LegacyPolicyMapping.id)) ).scalar_one_or_none() if mapping_id is not None: return LegacyPolicyPersistenceResult( diff --git a/policyengine_api/data/v2/policies/persistence.py b/policyengine_api/data/v2/policies/persistence.py index 2e7786d8b..3f7ca60b7 100644 --- a/policyengine_api/data/v2/policies/persistence.py +++ b/policyengine_api/data/v2/policies/persistence.py @@ -7,7 +7,7 @@ from uuid import UUID, uuid4 from sqlalchemy.dialects.postgresql import insert -from sqlmodel import Session, select +from sqlmodel import Session, col, select from policyengine_api.data.v2.models import ( ParameterValue, @@ -55,7 +55,7 @@ def _insert_policy( content_hash=content.content_hash, ) .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") - .returning(Policy.id) + .returning(col(Policy.id)) ) return session.execute(statement).scalar_one_or_none() @@ -97,20 +97,22 @@ def _stored_policy_command( values = session.exec( select(ParameterValue).where(ParameterValue.policy_id == policy.id) ).all() - return ResolvedPolicyCreateCommand( - country_id=policy.country_id, - tax_benefit_model_id=policy.tax_benefit_model_id, - tax_benefit_model_version_id=policy.tax_benefit_model_version_id, - policyengine_version=model_version.version, - parameter_values=[ - { - "parameter_id": value.parameter_id, - "value": value.value_json, - "start_date": value.start_date, - "end_date": value.end_date, - } - for value in values - ], + return ResolvedPolicyCreateCommand.model_validate( + { + "country_id": policy.country_id, + "tax_benefit_model_id": policy.tax_benefit_model_id, + "tax_benefit_model_version_id": policy.tax_benefit_model_version_id, + "policyengine_version": model_version.version, + "parameter_values": [ + { + "parameter_id": value.parameter_id, + "value": value.value_json, + "start_date": value.start_date, + "end_date": value.end_date, + } + for value in values + ], + } ) diff --git a/policyengine_api/data/v2/policies/query.py b/policyengine_api/data/v2/policies/query.py index 5852a441e..59cc319df 100644 --- a/policyengine_api/data/v2/policies/query.py +++ b/policyengine_api/data/v2/policies/query.py @@ -7,7 +7,7 @@ from typing import Any from uuid import UUID -from sqlmodel import Session, select +from sqlmodel import Session, col, select from policyengine_api.data.v2.models import Parameter, ParameterValue, Policy @@ -56,12 +56,12 @@ def _parameter_values_by_policy( return {} rows = session.exec( select(ParameterValue, Parameter.name) - .join(Parameter, Parameter.id == ParameterValue.parameter_id) - .where(ParameterValue.policy_id.in_(policy_ids)) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where(col(ParameterValue.policy_id).in_(policy_ids)) .order_by( - Parameter.name, - ParameterValue.start_date, - ParameterValue.id, + col(Parameter.name), + col(ParameterValue.start_date), + col(ParameterValue.id), ) ).all() for value, parameter_name in rows: @@ -129,7 +129,9 @@ def list_policies( if tax_benefit_model_id is not None: statement = statement.where(Policy.tax_benefit_model_id == tax_benefit_model_id) rows = session.exec( - statement.order_by(Policy.created_at, Policy.id).offset(offset).limit(limit + 1) + statement.order_by(col(Policy.created_at), col(Policy.id)) + .offset(offset) + .limit(limit + 1) ).all() has_more = len(rows) > limit policies = rows[:limit] diff --git a/policyengine_api/data/v2/user_policies/api_schemas.py b/policyengine_api/data/v2/user_policies/api_schemas.py index a1467108d..d6c3b57b5 100644 --- a/policyengine_api/data/v2/user_policies/api_schemas.py +++ b/policyengine_api/data/v2/user_policies/api_schemas.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime -from typing import Annotated, Generic, Literal, TypeVar +from typing import Annotated, Any, Generic, Literal, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, StringConstraints @@ -90,7 +90,7 @@ class UserPolicyErrorResponse(StrictUserPolicyAPIModel): message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] -USER_POLICY_ERROR_RESPONSES = { +USER_POLICY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": UserPolicyErrorResponse, "description": "Association content or country selection is invalid.", diff --git a/policyengine_api/data/v2/user_policies/legacy.py b/policyengine_api/data/v2/user_policies/legacy.py index 88ec6fec4..9ba9c7c6c 100644 --- a/policyengine_api/data/v2/user_policies/legacy.py +++ b/policyengine_api/data/v2/user_policies/legacy.py @@ -10,7 +10,7 @@ from pydantic import Field from sqlalchemy.dialects.postgresql import insert -from sqlmodel import Session, select +from sqlmodel import Session, col, select from policyengine_api.data.v2.models import ( LegacyUserMapping, @@ -132,14 +132,14 @@ def resolve_legacy_user_id( user = User(primary_country=primary_country) session.add(user) session.flush() - inserted_user_id = session.execute( + inserted_user_id: UUID | None = session.execute( insert(LegacyUserMapping) .values( legacy_user_id=legacy_user_id, user_id=user.id, ) .on_conflict_do_nothing(index_elements=[LegacyUserMapping.legacy_user_id]) - .returning(LegacyUserMapping.user_id) + .returning(col(LegacyUserMapping.user_id)) ).scalar_one_or_none() if inserted_user_id is not None: return inserted_user_id @@ -318,7 +318,7 @@ def persist_legacy_user_policy( .on_conflict_do_nothing( constraint="uq_legacy_user_policy_mappings_country_legacy" ) - .returning(LegacyUserPolicyMapping.id) + .returning(col(LegacyUserPolicyMapping.id)) ).scalar_one_or_none() if mapping_id is not None: return LegacyUserPolicyPersistenceResult( diff --git a/policyengine_api/data/v2/user_policies/query.py b/policyengine_api/data/v2/user_policies/query.py index 6c910bf10..c051be0d4 100644 --- a/policyengine_api/data/v2/user_policies/query.py +++ b/policyengine_api/data/v2/user_policies/query.py @@ -6,7 +6,7 @@ from datetime import datetime from uuid import UUID -from sqlmodel import Session, select +from sqlmodel import Session, col, select from policyengine_api.data.v2.models import UserPolicy @@ -102,7 +102,7 @@ def list_user_policies( if policy_id is not None: statement = statement.where(UserPolicy.policy_id == policy_id) rows = session.exec( - statement.order_by(UserPolicy.created_at, UserPolicy.id) + statement.order_by(col(UserPolicy.created_at), col(UserPolicy.id)) .offset(offset) .limit(limit + 1) ).all() diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index f8fab68fa..2d06c3ef8 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -4,12 +4,41 @@ from collections.abc import Callable from dataclasses import dataclass +from datetime import datetime from functools import lru_cache from importlib import metadata as importlib_metadata -from typing import Protocol +from typing import TYPE_CHECKING, Protocol +from uuid import UUID from policyengine_api.json_types import JSONObject +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.schemas import ( + MetadataCanonicalParameterValue, + MetadataDataset, + MetadataDetailResult, + MetadataEconomyOptionsResult, + MetadataModel, + MetadataModelSelectionResult, + MetadataModelVersionDetail, + MetadataPageResult, + MetadataParameterChild, + MetadataParameterSummary, + MetadataRegion, + MetadataVariable, + ) + from policyengine_api.data.v2.policies.query import PolicyPage, PolicyRead + from policyengine_api.data.v2.policies.schemas import NativePolicyCreateCommand + from policyengine_api.data.v2.user_policies.query import ( + UserPolicyPage, + UserPolicyRead, + ) + from policyengine_api.data.v2.user_policies.schemas import ( + UserPolicyCreateCommand, + UserPolicyPatchCommand, + ) + from policyengine_api.services.v2.policy_service import NativePolicyCreation + class MetadataReader(Protocol): """Read the already-loaded metadata document for a country.""" @@ -22,29 +51,213 @@ class V2MetadataResourceReader(Protocol): def close(self) -> None: ... + def list_models( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataModel]": ... + + def get_model( + self, + country_id: str, + model_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataModel]": ... + + def get_model_by_country( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> "MetadataModelSelectionResult": ... + + def list_model_versions( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataModelVersionDetail]": ... + + def get_model_version( + self, + country_id: str, + version_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataModelVersionDetail]": ... + + def list_variables( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + search: str | None = None, + ) -> "MetadataPageResult[MetadataVariable]": ... + + def get_variable( + self, + country_id: str, + variable_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataVariable]": ... + + def list_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + search: str | None = None, + ) -> "MetadataPageResult[MetadataParameterSummary]": ... + + def get_parameter( + self, + country_id: str, + parameter_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataParameterSummary]": ... + + def list_parameter_children( + self, + country_id: str, + policyengine_version: str | None = None, + *, + parent_path: str = "", + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataParameterChild]": ... + + def list_parameter_values( + self, + country_id: str, + policyengine_version: str | None = None, + *, + parameter_id: UUID | None = None, + current: bool = False, + offset: int = 0, + limit: int = 100, + now: datetime | None = None, + ) -> "MetadataPageResult[MetadataCanonicalParameterValue]": ... + + def get_parameter_value( + self, + country_id: str, + value_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataCanonicalParameterValue]": ... + + def list_datasets( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataDataset]": ... + + def get_dataset( + self, + country_id: str, + dataset_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataDataset]": ... + + def list_regions( + self, + country_id: str, + policyengine_version: str | None = None, + *, + region_type: str | None = None, + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataRegion]": ... + + def get_region( + self, + country_id: str, + region_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataRegion]": ... + + def get_region_by_code( + self, + country_id: str, + region_code: str, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataRegion]": ... + + def get_economy_options( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> "MetadataEconomyOptionsResult": ... + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" - def create_policy(self, command: object) -> object: ... + def create_policy( + self, + command: "NativePolicyCreateCommand", + ) -> "NativePolicyCreation": ... - def get_policy(self, *, country_id: str, policy_id: object) -> object: ... + def get_policy(self, *, country_id: str, policy_id: UUID) -> "PolicyRead": ... - def list_policies(self, **filters: object) -> object: ... + def list_policies( + self, + *, + country_id: str, + tax_benefit_model_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> "PolicyPage": ... class V2UserPolicyResourceService(Protocol): """Route-independent native association operations for one request.""" - def create_user_policy(self, command: object) -> object: ... - - def get_user_policy(self, **identity: object) -> object: ... - - def list_user_policies(self, **filters: object) -> object: ... - - def patch_user_policy(self, **changes: object) -> object: ... - - def delete_user_policy(self, **identity: object) -> None: ... + def create_user_policy( + self, + command: "UserPolicyCreateCommand", + ) -> "UserPolicyRead": ... + + def get_user_policy( + self, + *, + country_id: str, + association_id: UUID, + ) -> "UserPolicyRead": ... + + def list_user_policies( + self, + *, + country_id: str, + user_id: UUID, + policy_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> "UserPolicyPage": ... + + def patch_user_policy( + self, + *, + country_id: str, + association_id: UUID, + command: "UserPolicyPatchCommand", + ) -> "UserPolicyRead": ... + + def delete_user_policy( + self, + *, + country_id: str, + association_id: UUID, + ) -> None: ... class SimulationGatewayProbe(Protocol): diff --git a/policyengine_api/fastapi_routes/query_parameters.py b/policyengine_api/fastapi_routes/query_parameters.py index f8bd8a878..98670d410 100644 --- a/policyengine_api/fastapi_routes/query_parameters.py +++ b/policyengine_api/fastapi_routes/query_parameters.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Awaitable, Callable from inspect import Parameter, Signature from typing import Annotated, TypeVar @@ -31,7 +31,7 @@ def _duplicate_error(error: DuplicateScalarQueryParameterError) -> dict[str, obj def query_dependency( model_type: type[QueryParametersT], -) -> Callable[..., QueryParametersT]: +) -> Callable[..., Awaitable[QueryParametersT]]: """Build a typed dependency with runtime and OpenAPI query metadata.""" async def dependency( diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 5b5c447ae..0cedad8b1 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Callable -from typing import TypeVar +from typing import Any, TypeVar from pydantic import BaseModel from starlette.responses import JSONResponse @@ -18,10 +18,13 @@ ) from policyengine_api.data.v2.catalog.schemas import MetadataErrorResponse from policyengine_api.data.v2.settings import V2ConfigurationError -from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2MetadataResourceReader, +) -ERROR_RESPONSES = { +ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": MetadataErrorResponse, "description": "The resource request or PolicyEngine.py version is invalid.", @@ -63,7 +66,7 @@ def error_response(status_code: int, message: str) -> JSONResponse: def read_resource( dependencies: NativeRouteDependencies, response_type: type[ResponseT], - operation: Callable[[object], object], + operation: Callable[[V2MetadataResourceReader], object], ) -> ResponseT | JSONResponse: reader = None try: diff --git a/policyengine_api/fastapi_routes/v2/policy_routes.py b/policyengine_api/fastapi_routes/v2/policy_routes.py index f3efe57d3..6cc1f4761 100644 --- a/policyengine_api/fastapi_routes/v2/policy_routes.py +++ b/policyengine_api/fastapi_routes/v2/policy_routes.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Callable +from typing import TypeVar from uuid import UUID from fastapi import APIRouter, Depends, Request @@ -85,7 +86,12 @@ def _service_factory( return _default_v2_policy_service_factory -def _policy_operation(operation: Callable[[], object]) -> object | JSONResponse: +OperationT = TypeVar("OperationT") + + +def _policy_operation( + operation: Callable[[], OperationT], +) -> OperationT | JSONResponse: try: return operation() except PolicyCatalogValidationError as error: diff --git a/policyengine_api/fastapi_routes/v2/user_policy_routes.py b/policyengine_api/fastapi_routes/v2/user_policy_routes.py index 16b23fb8f..ad4cbf43d 100644 --- a/policyengine_api/fastapi_routes/v2/user_policy_routes.py +++ b/policyengine_api/fastapi_routes/v2/user_policy_routes.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Callable +from typing import TypeVar from uuid import UUID from fastapi import APIRouter, Depends @@ -58,9 +59,12 @@ def _service_factory( return _default_v2_user_policy_service_factory +OperationT = TypeVar("OperationT") + + def _association_operation( - operation: Callable[[], object], -) -> object | JSONResponse: + operation: Callable[[], OperationT], +) -> OperationT | JSONResponse: try: return operation() except AssociationCountryConflictError as error: diff --git a/pyproject.toml b/pyproject.toml index 5efe10516..e14d47a2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ dependencies = [ dev = [ "build", "coverage", + "mypy>=1.15,<2", "pytest", "pytest-snapshot", "pytest-timeout", @@ -73,6 +74,24 @@ packages = ["policyengine_api"] [tool.ruff.lint.per-file-ignores] "migrations/*/versions/*.py" = ["F401"] +[tool.mypy] +python_version = "3.11" +files = [ + "policyengine_api/query_parameters.py", + "policyengine_api/fastapi_routes/query_parameters.py", + "policyengine_api/fastapi_routes/dependencies.py", + "policyengine_api/fastapi_routes/v2", + "policyengine_api/data/v2/policies", + "policyengine_api/data/v2/user_policies", + "policyengine_api/services/v2", +] +explicit_package_bases = true +follow_imports = "silent" +disallow_untyped_defs = true +warn_return_any = true +warn_unused_ignores = true +show_error_codes = true + [tool.towncrier] package = "policyengine_api" directory = "changelog.d" diff --git a/uv.lock b/uv.lock index 886b8790b..49e9c2757 100644 --- a/uv.lock +++ b/uv.lock @@ -2,9 +2,12 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -1698,6 +1701,122 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "librt" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/9b/356320fbae2ac8467e21c5e73e1389c80468e4998c62cc7d3536cc51b614/librt-0.15.0.tar.gz", hash = "sha256:4e66cbe84437497d951b799d3e1551291b6fb3d643820a7014b3655d57a59162", size = 214338, upload-time = "2026-08-07T10:49:42.663Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/52/06790ced2ac7117f890c21bda43c39c958ec82aa665c0718e821d33ff939/librt-0.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:823b92cf3c18ecd08afc70c42473888b41b6e8ef5046f3b82c05c154a2fa3d22", size = 148039, upload-time = "2026-08-07T10:46:41.165Z" }, + { url = "https://files.pythonhosted.org/packages/e7/1d/8e150b7fc449a1f33c8a760965cc1f43b14fc1577d9d0b50ab2701420e74/librt-0.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c70bc1b602cf59917e8f0c7a2cbc8bcc6fbc14d5486136b00707a79619121d63", size = 153067, upload-time = "2026-08-07T10:46:42.418Z" }, + { url = "https://files.pythonhosted.org/packages/51/87/a162bc5a66a35599dc619ecb215145f4de7d68e886b479b6d12593139f7c/librt-0.15.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:814ff83a25b5fce8b9c80c4dd803153fb5c5599fc74db9e022466938368957ef", size = 493087, upload-time = "2026-08-07T10:46:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/e5/3a/aeea1fc620cf48060d3065b37614edbf97043c099d0f50782bc8ca61d897/librt-0.15.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:57f5eeb6ad4c180de583b1038e61fe5fbd9796bb69a8a1c1a0c7ddbec4c8c60f", size = 485608, upload-time = "2026-08-07T10:46:45.038Z" }, + { url = "https://files.pythonhosted.org/packages/52/ff/fe571ad416f0856fd0d5578ffc2e6dc531891e586e36b647bcf50569cab8/librt-0.15.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82909c8f7eb9952656b65d3147afde4cf8e6d5a991eebc86418b5e65843b0ab8", size = 498723, upload-time = "2026-08-07T10:46:46.35Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e1/7a65eb5dedb1f00aebd948cdd8e17add48bf066cab3514e9daf84ab45a6c/librt-0.15.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f779070399f991400fc451719e0ea388eb7de313388bada2c127a35de05f798a", size = 516002, upload-time = "2026-08-07T10:46:47.599Z" }, + { url = "https://files.pythonhosted.org/packages/5f/45/59832b0ebfbd08c2742e6ece372ceb53f18bf1faef5d33c8daf3abebf749/librt-0.15.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bac89069bc496ebdf4f79ebb57bbd10d0b214c8454225deb672d91002bd17e18", size = 508607, upload-time = "2026-08-07T10:46:48.873Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0d/37fa73f3b43ebd8259f91ae9102a15e5a54e65d581e48dea72df3e81d7a4/librt-0.15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e0d00c708fb2f5822b152429b1ac80a58dbbbc3f6c232c4d13a3f7fcf2ea5b4c", size = 530422, upload-time = "2026-08-07T10:46:50.45Z" }, + { url = "https://files.pythonhosted.org/packages/26/02/e046c6fe7a5881ac34623242192f484426ba8a75595fd18f22c53a3f530f/librt-0.15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6c6624fe268625869485553dd7cc1daf30d22558215bb2a4ff16f67a9801a31a", size = 534303, upload-time = "2026-08-07T10:46:51.693Z" }, + { url = "https://files.pythonhosted.org/packages/95/32/d5e6d861ab0366f3edf74f887ab0c9eb9f535aaf01d32b80b4f734daa179/librt-0.15.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f56b397858a23dacf35ede366ed2212fdc03a6a57a1ad36468ad6e9dc5fac091", size = 536084, upload-time = "2026-08-07T10:46:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/2a/de/d69d725513fe53fc90c6d7a1f86e4428939bad2fb905b17fe4c18d413dde/librt-0.15.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4388184646efe2054911c5b00a1077d6d1ee86a95b7e8ba96dc7850a809f3f40", size = 514307, upload-time = "2026-08-07T10:46:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/36/93/f8aded0d6682b4f25820fa86e0690f87f01df9fd7bd09ddb04d9167ad021/librt-0.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:97335f59082f9fe2ce6c2a9cc6433a0114bbb6cd4d5c09dd76c95c68b9f9a8b0", size = 557686, upload-time = "2026-08-07T10:46:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/74/09/ffeb6bdeb6cd862b4272fddc8ad05f938dd25d020ed517e631813917d80a/librt-0.15.0-cp311-cp311-win32.whl", hash = "sha256:83380ffde38062a2e9bb55d83e74474f6614665528b98a6928720fc006dfffbb", size = 104917, upload-time = "2026-08-07T10:46:56.605Z" }, + { url = "https://files.pythonhosted.org/packages/96/28/7e2313a3ffbf0b4de7ba3da58a09e488507b4bd1ea2b5e69378354a23415/librt-0.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:f75720477ee05d509a310e856cacc8d909adc182f7b91193c207bcc26d7ee6db", size = 125886, upload-time = "2026-08-07T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/39/9e/04b8c3cde014ef255ee785730425268354543acc38902093a40afa0dc164/librt-0.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:256237037a3ab001ae8d9803b2d43562a4c3aa38739843694349e4d5ebb0fd56", size = 111885, upload-time = "2026-08-07T10:46:58.787Z" }, + { url = "https://files.pythonhosted.org/packages/ba/39/99c25030e782bdfb7a21be8c05254806a2e4bbb05c8d50c2a2130acbfa05/librt-0.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e87bc679f86a99aa3b26e3c78eeb821a247c9a28eae48eaafcc32c3bf4c3bb9e", size = 151021, upload-time = "2026-08-07T10:47:00.057Z" }, + { url = "https://files.pythonhosted.org/packages/14/43/f4b1bd1b2888798a1409808889a25ea1ba49eaabce7d681ed27734c2df9d/librt-0.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:71599e011ac880e8e45d46047d714871894c7d4ab6f25626f8d4f89da21f368d", size = 155267, upload-time = "2026-08-07T10:47:01.311Z" }, + { url = "https://files.pythonhosted.org/packages/0c/db/3ad9c965c72f1e1d6beeec44ec10a54e17be8ae042fbb4baade16cbadced/librt-0.15.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c802434092b769b1d613ed2e13fac15fbfce1934a74bd10283b03c0fae231cd1", size = 503136, upload-time = "2026-08-07T10:47:02.45Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/5888a6d76acd62ebce66c61b74d94e9370b9c32929f111e487bb6546f8ed/librt-0.15.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5500eeae393a184d14e1f35645962c27129d20c81afa4069e6ef826ebc2b3aaa", size = 496670, upload-time = "2026-08-07T10:47:03.675Z" }, + { url = "https://files.pythonhosted.org/packages/29/39/ab57cc2f5b276156da02bb7f5a8921bada1cb1993ffec99acf811c602c23/librt-0.15.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6ecfc32dfb46fb7b565bcd6abf9412acf978775a998273d22888a6d7953730dd", size = 513688, upload-time = "2026-08-07T10:47:04.981Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b9/bdbb0b648b5c2befb031f4c6f3b1dd857415e8fb492a25a3c764a6681e6c/librt-0.15.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cc46cfd15022e35084355478c9ac809d90b1152222706ac9a7655ec21df6fa", size = 531904, upload-time = "2026-08-07T10:47:06.211Z" }, + { url = "https://files.pythonhosted.org/packages/93/26/473c2e4b6c104e9e58e27ce95fc8005c8bd4fc36cae4f254371125a92db8/librt-0.15.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5f51401d102c885b9ca509e62c79b1dbff286e1b9b047fde6f763780789356d", size = 524427, upload-time = "2026-08-07T10:47:07.592Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/03b3abb82b41714671b907bf6989b228e31e6a8af52dec82b5b0728dc250/librt-0.15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cc30523e3f1a23fb7511cc659834a0d01a1042bb9de359bc1c131cc4ec6c9656", size = 543155, upload-time = "2026-08-07T10:47:08.866Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0e/9bb1f0a4affbd0a1888f4f79dc03ed2a299d9a2c26c59ab2a97dcbf11903/librt-0.15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:59fe030d8ae4a57e3fb7756bf35a858de74e04066fc8555c53d0af979132af81", size = 546890, upload-time = "2026-08-07T10:47:10.327Z" }, + { url = "https://files.pythonhosted.org/packages/dc/84/6937a280d461f7de6e031ffb02edc2b7c3c90d49d630565ce8ff27cbc5f2/librt-0.15.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5a6526a2a956bbb1e4ae3568c82e650fc99119c66bb011ea60715744955a2b4d", size = 555163, upload-time = "2026-08-07T10:47:11.798Z" }, + { url = "https://files.pythonhosted.org/packages/bc/95/2a2853c1ee014bf102116e7f897a04beeaeb2461b45b79af98bdfb95f1ef/librt-0.15.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:85ea21ec6730194d67156b0e0b5430ccb1d61f8b8b907e39b37f9812b74a13f0", size = 535812, upload-time = "2026-08-07T10:47:13.279Z" }, + { url = "https://files.pythonhosted.org/packages/c9/4c/cf9601c1b4c5f09280acd5d83abdb2e68527a2be8257136eb42304218622/librt-0.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1e47b8ba865d7ede071a91a7163073bbaeb72541f1ef8a07d512c45c7b5007f2", size = 573688, upload-time = "2026-08-07T10:47:14.727Z" }, + { url = "https://files.pythonhosted.org/packages/47/6d/9ac7cbec46189a7625af4b5acbd25f10d827f4141b2002181848c8418923/librt-0.15.0-cp312-cp312-win32.whl", hash = "sha256:a5207ec414d1c4a2a7231b2086970dc036f94293cdf338190984958a013a42f1", size = 106138, upload-time = "2026-08-07T10:47:15.973Z" }, + { url = "https://files.pythonhosted.org/packages/38/d0/2ae99c83be86ce23f925ac1aeeedc777e97f427c4a8d190c70d0a16e9a87/librt-0.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:73b30cfa976659b3917c8f6153bdb0591c6a9ec6583599fd24a689b690622022", size = 126974, upload-time = "2026-08-07T10:47:17.049Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ef/dd24f9635c730b86b87587967dda7516b1845e8b17684603d31607fed598/librt-0.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:a54cf9e0ef47b96af580849db5471142200568ce1e02cbf416addab551369570", size = 112292, upload-time = "2026-08-07T10:47:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/e7/42/467b53a601b406ccd7b97c1fd54b59cb34f9185ad5ce7e9d5c3c4e8961c8/librt-0.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:db13ca398005abcbe538deda87b686d9bd08b7001cf40c4c06b444960ae10a26", size = 151029, upload-time = "2026-08-07T10:47:19.312Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/36c2299b7a94b84fdd01220d8a777a71be5be0925bb0dbdf71c0a06a34d9/librt-0.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa1f1995789dca3698bc550aaceb09a51bd5df0a057ff84ff15296cd1975b801", size = 155194, upload-time = "2026-08-07T10:47:20.398Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/ed5071f9325845e670bd36012757419767fbf56af77ed483077b9e4db541/librt-0.15.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55456ea87d8df21808446d03817be2f65e20391c1c615d9187440dff28cd08dc", size = 502568, upload-time = "2026-08-07T10:47:21.652Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/6450c67c3615d87704bcbc21323fafc69c799b06a044c447529f725d4b01/librt-0.15.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5a86a5a08c2235316bdb359d5dbb6ce0abfca7fac06363103e2c5af571d92f95", size = 496153, upload-time = "2026-08-07T10:47:22.925Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d6/5f52b722bc75076954b3bfd49be15ea362df4d580c6fb315d0f617100d30/librt-0.15.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e56b6a368529bed262da40ce13f8fef590db0479819cca84f16a1f01ac356d0b", size = 513336, upload-time = "2026-08-07T10:47:24.213Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e2/c08fd1d36ce63ea5a12b85c5d37f4550b5f86a692167e41e5a74222607ae/librt-0.15.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:234d8d394721fa0d786af15ebf1f3fb7f3ed82fd1cd0cde45c2f247b5d4281d2", size = 531661, upload-time = "2026-08-07T10:47:25.507Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d8/d9482fcbeb177b9eb87bb3899eeb3b42be690313c652f9e146b1d0681fb2/librt-0.15.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d8363d7accb0286ac3a0e633f396e93800dafb8150494505daf9515bbda591f3", size = 524487, upload-time = "2026-08-07T10:47:26.79Z" }, + { url = "https://files.pythonhosted.org/packages/10/cc/075171517b41f861753034fbb151b42cfc83bcc853849f24f5e66fd60ccf/librt-0.15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0f0ee3644d951f31055ad07d77d92520e84505dd7a432cc4cd501dd70ee06785", size = 543201, upload-time = "2026-08-07T10:47:27.999Z" }, + { url = "https://files.pythonhosted.org/packages/b0/03/42c2330f37eeb475b6affeedd06518f60035f323af3a839335e3fc9fef2d/librt-0.15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2cfd1a81a648806e6a7717be4cc4d1bb392fa229752bf8444ba365e381e984d6", size = 546467, upload-time = "2026-08-07T10:47:29.396Z" }, + { url = "https://files.pythonhosted.org/packages/57/1e/1ad4c5638f7e64d8560328bd25c54b409a661bdb6ff254b38ff90744288d/librt-0.15.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a6cd22c9da0d866558e46a041f1cc0c2bbb26b61b137b2347fa834c332e1d101", size = 555139, upload-time = "2026-08-07T10:47:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/49/41/39fa7d15db1204cd1cbe6514680fbdc243adf754a0885061308f43afc013/librt-0.15.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:6d5225ef8801e4ea5e482fa9b5dfb891dd9ef6f6d870f1f25d449ca2c70ac218", size = 536050, upload-time = "2026-08-07T10:47:32.222Z" }, + { url = "https://files.pythonhosted.org/packages/1e/88/c6dcf0dd8e26dc0c9a499a2abab8646c86dcaf9ecea9524cb46d3686331a/librt-0.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d28a05796b99f749bf8794f17ba9ba1612d0076b802e9cfc62c554634e9ce3b", size = 573700, upload-time = "2026-08-07T10:47:33.527Z" }, + { url = "https://files.pythonhosted.org/packages/1b/9b/ab54c71a7918a7c34fa5327fb61390a77446a07a146fbfb1165250a61035/librt-0.15.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2067ff438048cead9d223ca5675bae2a25e520a7c3e6c1498bf9c6892d22caab", size = 82194, upload-time = "2026-08-07T10:47:34.835Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b2/4f9a243bb892395f3becb80789ade13771701091f9f07ab8230247953ba8/librt-0.15.0-cp313-cp313-win32.whl", hash = "sha256:1cd3b721f24c206398b9e26da3c3a9c011e6e89d06f318ba8ebefc30f1003890", size = 106231, upload-time = "2026-08-07T10:47:36.251Z" }, + { url = "https://files.pythonhosted.org/packages/bf/af/64aff4885a40b93132382f2c314647d722574605416504379184ef3045ea/librt-0.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:f395a4a9a03ac062dbe9a9f82e0c720502e590a38feee6a757bc82e9c63afbd8", size = 126996, upload-time = "2026-08-07T10:47:37.453Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/335bccf6c7cb9028cb0b54aead27d9ece3f01f83bc6baa2abace5da655c1/librt-0.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:0a15cb554761247d84a3ec0cbdf4078d70725384f0e4662c0fa3b26266eb60ad", size = 112188, upload-time = "2026-08-07T10:47:38.729Z" }, + { url = "https://files.pythonhosted.org/packages/a8/93/949053fb462eecc4a9a5ee770a81f4b40be7b79538b245545d4aebc6b58b/librt-0.15.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f5de7feedc56337a088eb15cd9fafa9938367362221d8cc62c642b7f94821993", size = 149833, upload-time = "2026-08-07T10:47:39.86Z" }, + { url = "https://files.pythonhosted.org/packages/61/ca/8281aa6cd560a3420e4497729f6b704b53be3eeaaef82d5aeadddaf7441f/librt-0.15.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c0eb900c0e91f4aebe680845242e614f1864edfd44106380d0752ac29522bf8", size = 154088, upload-time = "2026-08-07T10:47:41.065Z" }, + { url = "https://files.pythonhosted.org/packages/dd/02/1a1662dceaba6a086360891448d5ce9a7d3555976cae59a31a39d744b9c7/librt-0.15.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8c9a650a188e38bac005048cbe6342e81407782944d01934540ab75e417df21", size = 494215, upload-time = "2026-08-07T10:47:42.388Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/99211619dc656370a3740c33d2b0b6d5a3fb1e73689314f6ed477a397dc4/librt-0.15.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:92bfed8deec93df30286b9fe9e3b1dd17329cc076a192b4ee5ec223841d54953", size = 491173, upload-time = "2026-08-07T10:47:43.683Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/5448d0b05f4579b635d3899176817ebf561af0e57bacd425b5b1887264c1/librt-0.15.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec4b19788f835711a2072f9dbe6b03b3bf32ed1f0fb30cf399bdd59d9f0c33fa", size = 505512, upload-time = "2026-08-07T10:47:45.314Z" }, + { url = "https://files.pythonhosted.org/packages/95/82/01940e40b83c43a546c4a3c896cf34ca272a9690899d55914e4827b3dcce/librt-0.15.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4c7bacb70930f3d0a56f4ecf1be474a1f0d941b01dd73b756f3c256d42cb879", size = 523073, upload-time = "2026-08-07T10:47:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/88/fa/759c0030f3ee371439eb26de34fc745807caf0abb878af7af4b8b7c3dd3d/librt-0.15.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3e79f05e4a08b4d880342673312bbc895b56df7765605796f15902eb5367d3ae", size = 515080, upload-time = "2026-08-07T10:47:48.319Z" }, + { url = "https://files.pythonhosted.org/packages/0b/27/894e072228fcb159703c655da69f8cd10dbed489c36e3df7dd032a2483be/librt-0.15.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a417149c0cba4d50b61e992e5a15e69eaf96746609b461cc4ed168aeef6b79dd", size = 534164, upload-time = "2026-08-07T10:47:49.875Z" }, + { url = "https://files.pythonhosted.org/packages/98/a3/0078e91c1f36f8815db17827de15650b9a3fe56c55fbf998c854b34e40d3/librt-0.15.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:da7a94d6a3411f579d72aa3e3bc5fbca7ed4549f3dbd7e5de3aa567333374285", size = 540616, upload-time = "2026-08-07T10:47:51.408Z" }, + { url = "https://files.pythonhosted.org/packages/86/33/81a29b796dd52a45e9ef7974c7732926e8f10f15b8d2be505665979f896d/librt-0.15.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:856f743ae607f2c1380eccb566c0038a9fb3eabf0fc2be2704d76d9f73557239", size = 545890, upload-time = "2026-08-07T10:47:52.818Z" }, + { url = "https://files.pythonhosted.org/packages/05/82/8be1baa1350e5d30cfd70ae79d0a6f4dc5862ef47f7bb2808aabc9bb86e5/librt-0.15.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:779a6e7c894737e5983e7790a9c78c4000c30e23c9aada08081bdbea53b0fa60", size = 523287, upload-time = "2026-08-07T10:47:54.165Z" }, + { url = "https://files.pythonhosted.org/packages/c6/4f/d1be6a01a35c20ef734e0e44113f87d4af756a9354a89dcfbe3b4f8af5e1/librt-0.15.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:96bb17dbe8bab3c0954fbebfc69ed395599de75b6bbc35e3270a878e15d4dd65", size = 565868, upload-time = "2026-08-07T10:47:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/67/88/649cfa33f5825927b160610f670bdab012a64d627eddb94fa795ea4292fd/librt-0.15.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:7220697efaa6e5348fc3d18ee7f8563d4bfecd9872b37ffb915bfc1d08840622", size = 81619, upload-time = "2026-08-07T10:47:56.886Z" }, + { url = "https://files.pythonhosted.org/packages/22/31/8e88a8d5e48fc8d1a817787fb6811dfff6499acd6c8683dd83934aa6ede0/librt-0.15.0-cp314-cp314-win32.whl", hash = "sha256:f54598964d357b1c5ab77cf5d92f21e598fe0e23cdbe9618480807f81b4eba15", size = 100138, upload-time = "2026-08-07T10:47:58.093Z" }, + { url = "https://files.pythonhosted.org/packages/80/92/20fd6c4b6a1b1a564b076d55cd3d427d8428217d7638dc25a654cc4791d4/librt-0.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:3ff5893a2c23d886aa9ce786de5ac6ddc74aeeaf90743682b74d920e117d2e28", size = 121258, upload-time = "2026-08-07T10:47:59.564Z" }, + { url = "https://files.pythonhosted.org/packages/fc/28/6af430b44d9ebb897b865a3c363b6dcace51357be2347cc0f8f869656a86/librt-0.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:3722a099730704c9a3d70c879fc0f51daec25fe5f1555672d97bc595abeafb95", size = 106467, upload-time = "2026-08-07T10:48:01.097Z" }, + { url = "https://files.pythonhosted.org/packages/7e/aa/b42bb798942ced219f6d63b27e07f91237887a8d0bd0921666db79a13790/librt-0.15.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:38c0c7d4b6fc06c3324b3f9162c8391bfc4fd9dde53afe1033ce7edb48d5a714", size = 159523, upload-time = "2026-08-07T10:48:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/75/03/1b53cd4ef904e73b1d828a5f90143bf94a2967d7cfff0b9ccf93e12aa9b4/librt-0.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8b2fdd7ead3c995c37940a790690660d0ca006c302db26cc51933f6766866fc3", size = 161638, upload-time = "2026-08-07T10:48:03.725Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c4/9f9c9fba097d49e9e694c2b4dc331df31884645ecbc58a93b4b5fc69d2c5/librt-0.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2fde98cf1fc4bac144ce23c2c4c017b924ba714509ea9334977b0b27050c837d", size = 701795, upload-time = "2026-08-07T10:48:05.135Z" }, + { url = "https://files.pythonhosted.org/packages/4c/05/0966840bda0380c8ae167b9043c6230202941cc90ea29c48e096964c765e/librt-0.15.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:e3b461183c5fa7681b48560f91515f53a953122fb30c71e07abc67d7ddf58c38", size = 682147, upload-time = "2026-08-07T10:48:06.555Z" }, + { url = "https://files.pythonhosted.org/packages/18/af/1c47ca573c30ea47d195aec26133af522fea1104afaace028d7b32247ea8/librt-0.15.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4bbcc257e3babea20a91715c361b24554ec4e8f51aa578568afc230799fe1a19", size = 696397, upload-time = "2026-08-07T10:48:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0f/1aed6223d4f9f9d1171a8596ff100ea4c3f7699fea7a4ba657c3e60daa6c/librt-0.15.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b845b8d48088fad0cadc84be4b8fda63203be7e9237b71015b3925443c1f35ab", size = 722542, upload-time = "2026-08-07T10:48:09.569Z" }, + { url = "https://files.pythonhosted.org/packages/c6/22/9e3a929aea456c97d69e6ef3884efea56d4807f97399471cc946baebd8af/librt-0.15.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b30e600e8f337b9bd7f39b86d9fdfedc73cc46e3d0f745931a23a234220bb7e2", size = 729709, upload-time = "2026-08-07T10:48:11.129Z" }, + { url = "https://files.pythonhosted.org/packages/e9/1b/c327ef6018e3a9ca0b8e7c5eddeeb331ba8f9b76c24e126d37d0f6d62faf/librt-0.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:64b0c8c35aa4c4ed79896359f3e0b285cbe4e610042106500da4811c322cc108", size = 752891, upload-time = "2026-08-07T10:48:12.558Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d1/d5f1ea02c56930087009e39db9b70660a663e76c730b27b925d786718457/librt-0.15.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0da0d94cb802f32a0524653e7201f2cef72d5f700a5407678f5290483d4fcd08", size = 745301, upload-time = "2026-08-07T10:48:14.55Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3c/5f7c585d15ebb2250c73e7c0ee4e9e47be72c65d520c07ddbcdc62037674/librt-0.15.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4a6369168d371207339b1e50d4532b06a7121586141f82599505a3f315751d47", size = 747921, upload-time = "2026-08-07T10:48:16.453Z" }, + { url = "https://files.pythonhosted.org/packages/7f/52/1443a446486eba966bcbca1696b472e4f210320ec42f490a47f48fbf0fdc/librt-0.15.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c434e072557ade9cbc642d052c89d031efe47d5c9614523619d0d74a02378e81", size = 727561, upload-time = "2026-08-07T10:48:18.089Z" }, + { url = "https://files.pythonhosted.org/packages/79/91/2270a9380f11725cf83ce1925a5e32dd1dde2be9bba597f25c10a38644e7/librt-0.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7eec6a42018bc1d45763b1c162d3d2bf7c3b9a1b0ed30d3e91dcba390efefcc", size = 774417, upload-time = "2026-08-07T10:48:19.611Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/f4b1548d4f5b99186737fe27aec238e9823e8d5d23bf4df007c030689dc5/librt-0.15.0-cp314-cp314t-win32.whl", hash = "sha256:6912fa5e635d74529ac7cdb1bdf6ca3af4453da8d1edbe0110ee1cb4ad407ebf", size = 104381, upload-time = "2026-08-07T10:48:21.048Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/134afad262def1de04c0843c376d02135f1168af43f22e09a52bd8394727/librt-0.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8e11699ed745931c395acd3621b07062e0f840efa6935aad87a64ed0995f0915", size = 127034, upload-time = "2026-08-07T10:48:22.561Z" }, + { url = "https://files.pythonhosted.org/packages/99/5f/1b6846b20572bd699c9e9ec321a5f781845bee477df2aa2a43b28bc40119/librt-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5d2a91724463bfed4f573cd7a9fdc856d2e230d0c0e5a61416a93481dccd8605", size = 110827, upload-time = "2026-08-07T10:48:23.804Z" }, + { url = "https://files.pythonhosted.org/packages/c6/44/4de9f4ddadb009a55c7758eb5736d62534a7daaf27bd71bc50e64b606b06/librt-0.15.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:8443e38dcfcfdbcf5add5118c623efd788d65ac2e25756d6251a54a06a4d0aca", size = 149843, upload-time = "2026-08-07T10:48:25.148Z" }, + { url = "https://files.pythonhosted.org/packages/1f/eb/5d9ab71e30119c44094e0275f38b47dd327aea0f843a080396677029d508/librt-0.15.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6d15a29033c57490cfe2069097c6fc4049e4e65ffbb749be7dc453b7c4c68965", size = 154510, upload-time = "2026-08-07T10:48:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/8505d1b8f5e8c19587bd03f7429993b3e9ce5c06819d856bfb11d919374c/librt-0.15.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2c05c729b589e734c09578bf5964be48a911765484840d017bbc84f49d4c4ad", size = 497543, upload-time = "2026-08-07T10:48:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9a/3a8390775cb095765aded027ac9c63e7c8ea74e731498607544c6505de0e/librt-0.15.0-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:fa60887537e1d0cd2d9982269d33a709bf54b195cd2b9364fc0a758022af5bd9", size = 480452, upload-time = "2026-08-07T10:48:29.531Z" }, + { url = "https://files.pythonhosted.org/packages/e7/40/258a4a7117ee915d66de5cd9b8ade65a440993161107ce3a686f1859955c/librt-0.15.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d8bc24219b24c0af375718942ab75e3544b2763085f40f965be4326734ae8328", size = 507768, upload-time = "2026-08-07T10:48:31.007Z" }, + { url = "https://files.pythonhosted.org/packages/6b/c6/2f4dd296c97a0b85b98894519b279408ec9dd602d4f692b1ea0e25dee670/librt-0.15.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86a21a7bd3fe3a419512ef424cc1c020f6771d0b29cfddff36d1635a855e63f0", size = 525122, upload-time = "2026-08-07T10:48:32.7Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/29eab42be13b2bf0ea8cb227135a45d44693e30a7e8b92871981ff56b82b/librt-0.15.0-cp315-cp315-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dbab647e88d90b3167b91efe7091e248653688ed4337e4f90907a722c7361bb9", size = 520371, upload-time = "2026-08-07T10:48:34.294Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/4bad71adeca8fe208b775c2a35417fa5a2584c8f4791daaf89a89450fea1/librt-0.15.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d8edcf6f550e918dca779c069b9e156385c60b406f99fc7641f32c52f7193659", size = 537258, upload-time = "2026-08-07T10:48:35.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/63/59dba6143fdcc7240c54458b629f3250000a61b8945890fc9efd451b19c5/librt-0.15.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:8b62076030baa2d8b1501a46bf0e19c27a489aa90671c55665bff7887f7660b0", size = 527432, upload-time = "2026-08-07T10:48:37.466Z" }, + { url = "https://files.pythonhosted.org/packages/ec/21/21a24c6a2327d8362580efebe77286bf47b0f4062ec5ea41766e609d3c7d/librt-0.15.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:d00d20d1818e82a07a0ee0aa89a98b17ed7916b92441090b683719cb20a59b6d", size = 548108, upload-time = "2026-08-07T10:48:39.384Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6d/fc68c89a7971418b41f9a873623ff935cb864097544c6a2f8ce491c8ef5d/librt-0.15.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4e6ee93fc3cf848dcbf0cce2eca73d8e7dcd0cc2b6df3a529d57750b30a4c55c", size = 529681, upload-time = "2026-08-07T10:48:41.392Z" }, + { url = "https://files.pythonhosted.org/packages/65/7e/c2d98766124400d722063a630b0fde38a9fc768705d37eecca15c47dc192/librt-0.15.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:32896a0af72508ea979e0acb4e4c04cbeeae04938167950d535c83c45597167d", size = 567736, upload-time = "2026-08-07T10:48:43.124Z" }, + { url = "https://files.pythonhosted.org/packages/55/6c/f8c34a95e3a515c6e1c192b89511e7253c89a7760c6b500d57ffdb8d2dc8/librt-0.15.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:ec3ba415afaf951f6951b1dd16d3c8e4f540065fc382d7e70b823a79567ca374", size = 81673, upload-time = "2026-08-07T10:48:44.645Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/e23fa8e78679ec45728188650b39e8ff476c83b691c96f749217df3b1b7c/librt-0.15.0-cp315-cp315-win32.whl", hash = "sha256:d2813ba2503764f0450680c533d13df7cff9b49df1411062eded5f67db4195b9", size = 100081, upload-time = "2026-08-07T10:48:46.171Z" }, + { url = "https://files.pythonhosted.org/packages/e1/dc/3eb4c5e297343f0620a55532cd7c8d764d3001fa2159212dadf480464827/librt-0.15.0-cp315-cp315-win_amd64.whl", hash = "sha256:b87d67e33afaf265262f2a66db578284b88ee2e6fcd224579cb5c15518677ad8", size = 121228, upload-time = "2026-08-07T10:48:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/97/70/43abce19f04e49762f8ec834c8fafee13cc40fd6b94a72a24e534febfcd0/librt-0.15.0-cp315-cp315-win_arm64.whl", hash = "sha256:713bd7df21170b982e729e46870f31d6b437bd1a9b4648cffb529bd3c2ec5c4b", size = 106487, upload-time = "2026-08-07T10:48:49.095Z" }, + { url = "https://files.pythonhosted.org/packages/de/15/83f2deddb9368b8951ec8c9477269b5b9b8bd9bbf15e57402d0f38817dca/librt-0.15.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:3de789c82752730f94782a5ee518baf9c05edf85733aeaf73bb6e518755cdf54", size = 159448, upload-time = "2026-08-07T10:48:50.649Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/043097353f9b3c73b583d07f6b8e552795463f4bfc8caf85e42eee50c26a/librt-0.15.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:e0b5deec9a8664eb722c797241970fd4aa1894d25fda36a1ddac0f7407606bd6", size = 161686, upload-time = "2026-08-07T10:48:52.174Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2a/8ae77f9719d42ce71cd708560a3557b38ac3c17a0383e57f87084de45bbe/librt-0.15.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5563302a8359bc2295bb7084d1a8ed1519df96afb30eb2aa4e0bff7b54228988", size = 710668, upload-time = "2026-08-07T10:48:53.782Z" }, + { url = "https://files.pythonhosted.org/packages/61/34/c0436ea134deb9a0d6da80a396a2739a81cb31e0418f7227239e23140898/librt-0.15.0-cp315-cp315t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:22d6263b9d39d7bbb286fa791945646e3218f1be2d693e36fb630f1d0e59cd13", size = 679396, upload-time = "2026-08-07T10:48:55.645Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/001e0d99aa9250d5cd5715a9081291a20656083459f9019cda15255329e1/librt-0.15.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:39ffd14646190c454f0d86e0d256b33f00a87a26ab410e619773b841d0e41416", size = 704313, upload-time = "2026-08-07T10:48:57.46Z" }, + { url = "https://files.pythonhosted.org/packages/2d/53/b34fa9d0ff00f136f4d58ebb4c411ff634baed1eb412bb602a2bc8dcafcb/librt-0.15.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c47318cd3a61401452de11282242937e3e057c4fd3dbaf601e269d0928a06c0a", size = 729847, upload-time = "2026-08-07T10:48:59.231Z" }, + { url = "https://files.pythonhosted.org/packages/86/ac/fa4d7a424665040e95baf480a6d523446057684b6758624c85338e8a23b2/librt-0.15.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a56a1d4f859a82ca5b99fc4b82c9b027b15e3c455c5cd99e7d0719f27bb20b6c", size = 742736, upload-time = "2026-08-07T10:49:01.151Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/e17a9bb5de6fb8c3186ed1a7d68d21618b027ac2d3633e03d3b6109c67ae/librt-0.15.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:077471b3182db4e17c36ae91555f36a4d2c00080b267f749bcad34a478a9a302", size = 763454, upload-time = "2026-08-07T10:49:03.039Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ec/ecd02cd30935b931b9cdbfed6ab5a099c51b280b4e7baa274da80978ed27/librt-0.15.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:411ca4d1b905b860ceba7570dd6717a71dedaddcc4b0f77ece710aa41ee11f8d", size = 743296, upload-time = "2026-08-07T10:49:04.941Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b5/b3c2b8353ce820a4854f78d19321344242f89fa71c975b71132ba9bf242a/librt-0.15.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1256589e0b0adb31751d685a68bce29d73407ddf4ef05d4188f49d5dcf9566d9", size = 756217, upload-time = "2026-08-07T10:49:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/3c/52/6cc22542ba59146b05cca2a656f9ff8bb67e38e63d12c3b0cc183d837bf1/librt-0.15.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:f42b74a53e5f26a0ba0007411a7455b66c67ce4022a39cc1f56fc4efd65bcbab", size = 741934, upload-time = "2026-08-07T10:49:08.839Z" }, + { url = "https://files.pythonhosted.org/packages/40/32/a04b72b1aa86e3be23b2ecff8c1aad2dcc955bd3956d6d26e7e34267e57a/librt-0.15.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:291bf73caf78b9e88d6fae9bfd693207ff7d832e2fdbe2cf8e746bc13f5f892b", size = 783763, upload-time = "2026-08-07T10:49:10.661Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f0/89eb11dffbe9279ff37144dec786927314502ae0b114f1449dc78c458aab/librt-0.15.0-cp315-cp315t-win32.whl", hash = "sha256:c16d15ee371643ab48dc8248a3e680ebbeca573a13af2c3dd0c985b142d77162", size = 104313, upload-time = "2026-08-07T10:49:12.305Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4a/1f1978c200f563beda63c36adff2d65bbecb81e365e8e69e572f5f70fbc6/librt-0.15.0-cp315-cp315t-win_amd64.whl", hash = "sha256:dbd605739f228912dc49027cb764456b9757750bdc2b6b7773164db7096c6fd1", size = 126889, upload-time = "2026-08-07T10:49:13.881Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/800800bfed7b1fb10fc3f3d557785c3854e80d3f7a9800d784b176a1fc2d/librt-0.15.0-cp315-cp315t-win_arm64.whl", hash = "sha256:84d244b00604d17df3fc7736c327892d6bba66181254aa4087be807b6c342bdc", size = 110700, upload-time = "2026-08-07T10:49:15.499Z" }, +] + [[package]] name = "linecheck" version = "0.1.0" @@ -2013,6 +2132,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] +[[package]] +name = "mypy" +version = "1.20.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/af/e3d4b3e9ec91a0ff9aabfdb38692952acf49bbb899c2e4c29acb3a6da3ae/mypy-1.20.2.tar.gz", hash = "sha256:e8222c26daaafd9e8626dec58ae36029f82585890589576f769a650dd20fd665", size = 3817349, upload-time = "2026-04-21T17:12:28.473Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/4d/9ebeae211caccbdaddde7ed5e31dfcf57faac66be9b11deb1dc6526c8078/mypy-1.20.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4077797a273e56e8843d001e9dfe4ba10e33323d6ade647ff260e5cd97d9758c", size = 14371307, upload-time = "2026-04-21T17:08:56.442Z" }, + { url = "https://files.pythonhosted.org/packages/95/d7/93473d34b61f04fac1aecc01368485c89c5c4af7a4b9a0cab5d77d04b63f/mypy-1.20.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cdecf62abcc4292500d7858aeae87a1f8f1150f4c4dd08fb0b336ee79b2a6df3", size = 13258917, upload-time = "2026-04-21T17:05:50.978Z" }, + { url = "https://files.pythonhosted.org/packages/e2/30/3dd903e8bafb7b5f7bf87fcd58f8382086dea2aa19f0a7b357f21f63071b/mypy-1.20.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c566c3a88b6ece59b3d70f65bedef17304f48eb52ff040a6a18214e1917b3254", size = 13700516, upload-time = "2026-04-21T17:11:33.161Z" }, + { url = "https://files.pythonhosted.org/packages/07/05/c61a140aba4c729ac7bc99ae26fc627c78a6e08f5b9dd319244ea71a3d7e/mypy-1.20.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0deb80d062b2479f2c87ae568f89845afc71d11bc41b04179e58165fd9f31e98", size = 14562889, upload-time = "2026-04-21T17:05:27.674Z" }, + { url = "https://files.pythonhosted.org/packages/fd/87/da78243742ffa8a36d98c3010f0d829f93d5da4e6786f1a1a6f2ad616502/mypy-1.20.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bba9ad231e92a3e424b3e56b65aa17704993425bba97e302c832f9466bb85bac", size = 14803844, upload-time = "2026-04-21T17:10:06.2Z" }, + { url = "https://files.pythonhosted.org/packages/37/52/10a1ddf91b40f843943a3c6db51e2df59c9e237f29d355e95eaab427461f/mypy-1.20.2-cp311-cp311-win_amd64.whl", hash = "sha256:baf593f2765fa3a6b1ef95807dbaa3d25b594f6a52adcc506a6b9cb115e1be67", size = 10846300, upload-time = "2026-04-21T17:12:23.886Z" }, + { url = "https://files.pythonhosted.org/packages/20/02/f9a4415b664c53bd34d6709be59da303abcae986dc4ac847b402edb6fa1e/mypy-1.20.2-cp311-cp311-win_arm64.whl", hash = "sha256:20175a1c0f49863946ec20b7f63255768058ac4f07d2b9ded6a6b46cfb5a9100", size = 9779498, upload-time = "2026-04-21T17:09:23.695Z" }, + { url = "https://files.pythonhosted.org/packages/71/4e/7560e4528db9e9b147e4c0f22660466bf30a0a1fe3d63d1b9d3b0fd354ee/mypy-1.20.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4dbfcf869f6b0517f70cf0030ba6ea1d6645e132337a7d5204a18d8d5636c02b", size = 14539393, upload-time = "2026-04-21T17:07:12.52Z" }, + { url = "https://files.pythonhosted.org/packages/32/d9/34a5efed8124f5a9234f55ac6a4ced4201e2c5b81e1109c49ad23190ec8c/mypy-1.20.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b6481b228d072315b053210b01ac320e1be243dc17f9e5887ef167f23f5fae4", size = 13361642, upload-time = "2026-04-21T17:06:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/d1/14/eb377acf78c03c92d566a1510cda8137348215b5335085ef662ab82ecd3a/mypy-1.20.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34397cdced6b90b836e38182076049fdb41424322e0b0728c946b0939ebdf9f6", size = 13740347, upload-time = "2026-04-21T17:12:04.73Z" }, + { url = "https://files.pythonhosted.org/packages/b9/94/7e4634a32b641aa1c112422eed1bbece61ee16205f674190e8b536f884de/mypy-1.20.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5da6976f20cae27059ea8d0c86e7cef3de720e04c4bb9ee18e3690fdb792066", size = 14734042, upload-time = "2026-04-21T17:07:43.16Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f3/f7e62395cb7f434541b4491a01149a4439e28ace4c0c632bbf5431e92d1f/mypy-1.20.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:56908d7e08318d39f85b1f0c6cfd47b0cac1a130da677630dac0de3e0623e102", size = 14964958, upload-time = "2026-04-21T17:11:00.665Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0d/47e3c3a0ec2a876e35aeac365df3cac7776c36bbd4ed18cc521e1b9d255b/mypy-1.20.2-cp312-cp312-win_amd64.whl", hash = "sha256:d52ad8d78522da1d308789df651ee5379088e77c76cb1994858d40a426b343b9", size = 10911340, upload-time = "2026-04-21T17:10:49.179Z" }, + { url = "https://files.pythonhosted.org/packages/d6/b2/6c852d72e0ea8b01f49da817fb52539993cde327e7d010e0103dc12d0dac/mypy-1.20.2-cp312-cp312-win_arm64.whl", hash = "sha256:785b08db19c9f214dc37d65f7c165d19a30fcecb48abfa30f31b01b5acaabb58", size = 9833947, upload-time = "2026-04-21T17:09:05.267Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c4/b93812d3a192c9bcf5df405bd2f30277cd0e48106a14d1023c7f6ed6e39b/mypy-1.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:edfbfca868cdd6bd8d974a60f8a3682f5565d3f5c99b327640cedd24c4264026", size = 14524670, upload-time = "2026-04-21T17:10:30.737Z" }, + { url = "https://files.pythonhosted.org/packages/f3/47/42c122501bff18eaf1e8f457f5c017933452d8acdc52918a9f59f6812955/mypy-1.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e2877a02380adfcdbc69071a0f74d6e9dbbf593c0dc9d174e1f223ffd5281943", size = 13336218, upload-time = "2026-04-21T17:08:44.069Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/75bbc92f41725fbd585fb17b440b1119b576105df1013622983e18640a93/mypy-1.20.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7488448de6007cd5177c6cea0517ac33b4c0f5ee9b5e9f2be51ce75511a85517", size = 13724906, upload-time = "2026-04-21T17:08:01.02Z" }, + { url = "https://files.pythonhosted.org/packages/a1/32/4c49da27a606167391ff0c39aa955707a00edc500572e562f7c36c08a71f/mypy-1.20.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb9c2fa06887e21d6a3a868762acb82aec34e2c6fd0174064f27c93ede68ad15", size = 14726046, upload-time = "2026-04-21T17:11:22.354Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fc/4e354a1bd70216359deb0c9c54847ee6b32ef78dfb09f5131ff99b494078/mypy-1.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d56a78b646f2e3daa865bc70cd5ec5a46c50045801ca8ff17a0c43abc97e3ee", size = 14955587, upload-time = "2026-04-21T17:12:16.033Z" }, + { url = "https://files.pythonhosted.org/packages/62/b2/c0f2056e9eb8f08c62cafd9715e4584b89132bdc832fcf85d27d07b5f3e5/mypy-1.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:2a4102b03bb7481d9a91a6da8d174740c9c8c4401024684b9ca3b7cc5e49852f", size = 10922681, upload-time = "2026-04-21T17:06:35.842Z" }, + { url = "https://files.pythonhosted.org/packages/e5/14/065e333721f05de8ef683d0aa804c23026bcc287446b61cac657b902ccac/mypy-1.20.2-cp313-cp313-win_arm64.whl", hash = "sha256:a95a9248b0c6fd933a442c03c3b113c3b61320086b88e2c444676d3fd1ca3330", size = 9830560, upload-time = "2026-04-21T17:07:51.023Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d1/b4ec96b0ecc620a4443570c6e95c867903428cfcde4206518eafdd5880c3/mypy-1.20.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:419413398fe250aae057fd2fe50166b61077083c9b82754c341cf4fd73038f30", size = 14524561, upload-time = "2026-04-21T17:06:27.325Z" }, + { url = "https://files.pythonhosted.org/packages/3a/63/d2c2ff4fa66bc49477d32dfa26e8a167ba803ea6a69c5efb416036909d30/mypy-1.20.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e73c07f23009962885c197ccb9b41356a30cc0e5a1d0c2ea8fd8fb1362d7f924", size = 13363883, upload-time = "2026-04-21T17:11:11.239Z" }, + { url = "https://files.pythonhosted.org/packages/2a/56/983916806bf4eddeaaa2c9230903c3669c6718552a921154e1c5182c701f/mypy-1.20.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c64e5973df366b747646fc98da921f9d6eba9716d57d1db94a83c026a08e0fb", size = 13742945, upload-time = "2026-04-21T17:08:34.181Z" }, + { url = "https://files.pythonhosted.org/packages/19/65/0cd9285ab010ee8214c83d67c6b49417c40d86ce46f1aa109457b5a9b8d7/mypy-1.20.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a65aa591af023864fd08a97da9974e919452cfe19cb146c8a5dc692626445dc", size = 14706163, upload-time = "2026-04-21T17:05:15.51Z" }, + { url = "https://files.pythonhosted.org/packages/94/97/48ff3b297cafcc94d185243a9190836fb1b01c1b0918fff64e941e973cc9/mypy-1.20.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fef51b01e638974a6e69885687e9bd40c8d1e09a6cd291cca0619625cf1f558", size = 14938677, upload-time = "2026-04-21T17:05:39.562Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a1/1b4233d255bdd0b38a1f284feeb1c143ca508c19184964e22f8d837ec851/mypy-1.20.2-cp314-cp314-win_amd64.whl", hash = "sha256:913485a03f1bcf5d279409a9d2b9ed565c151f61c09f29991e5faa14033da4c8", size = 11089322, upload-time = "2026-04-21T17:06:44.29Z" }, + { url = "https://files.pythonhosted.org/packages/78/c2/ce7ee2ba36aeb954ba50f18fa25d9c1188578654b97d02a66a15b6f09531/mypy-1.20.2-cp314-cp314-win_arm64.whl", hash = "sha256:c3bae4f855d965b5453784300c12ffc63a548304ac7f99e55d4dc7c898673aa3", size = 10017775, upload-time = "2026-04-21T17:07:20.732Z" }, + { url = "https://files.pythonhosted.org/packages/4e/a1/9d93a7d0b5859af0ead82b4888b46df6c8797e1bc5e1e262a08518c6d48e/mypy-1.20.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2de3dcea53babc1c3237a19002bc3d228ce1833278f093b8d619e06e7cc79609", size = 15549002, upload-time = "2026-04-21T17:08:23.107Z" }, + { url = "https://files.pythonhosted.org/packages/00/d2/09a6a10ee1bf0008f6c144d9676f2ca6a12512151b4e0ad0ff6c4fac5337/mypy-1.20.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:52b176444e2e5054dfcbcb8c75b0b719865c96247b37407184bbfca5c353f2c2", size = 14401942, upload-time = "2026-04-21T17:07:31.837Z" }, + { url = "https://files.pythonhosted.org/packages/57/da/9594b75c3c019e805250bed3583bdf4443ff9e6ef08f97e39ae308cb06f2/mypy-1.20.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:688c3312e5dadb573a2c69c82af3a298d43ecf9e6d264e0f95df960b5f6ac19c", size = 15041649, upload-time = "2026-04-21T17:09:34.653Z" }, + { url = "https://files.pythonhosted.org/packages/97/77/f75a65c278e6e8eba2071f7f5a90481891053ecc39878cc444634d892abe/mypy-1.20.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29752dbbf8cc53f89f6ac096d363314333045c257c9c75cbd189ca2de0455744", size = 15864588, upload-time = "2026-04-21T17:11:44.936Z" }, + { url = "https://files.pythonhosted.org/packages/d7/46/1a4e1c66e96c1a3246ddf5403d122ac9b0a8d2b7e65730b9d6533ba7a6d3/mypy-1.20.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:803203d2b6ea644982c644895c2f78b28d0e208bba7b27d9b921e0ec5eb207c6", size = 16093956, upload-time = "2026-04-21T17:10:17.683Z" }, + { url = "https://files.pythonhosted.org/packages/5a/2c/78a8851264dec38cd736ca5b8bc9380674df0dd0be7792f538916157716c/mypy-1.20.2-cp314-cp314t-win_amd64.whl", hash = "sha256:9bcb8aa397ff0093c824182fd76a935a9ba7ad097fcbef80ae89bf6c1731d8ec", size = 12568661, upload-time = "2026-04-21T17:11:54.473Z" }, + { url = "https://files.pythonhosted.org/packages/83/01/cd7318aa03493322ce275a0e14f4f52b8896335e4e79d4fb8153a7ad2b77/mypy-1.20.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e061b58443f1736f8a37c48978d7ab581636d6ab03e3d4f99e3fa90463bb9382", size = 10389240, upload-time = "2026-04-21T17:09:42.719Z" }, + { url = "https://files.pythonhosted.org/packages/28/9a/f23c163e25b11074188251b0b5a0342625fc1cdb6af604757174fa9acc9b/mypy-1.20.2-py3-none-any.whl", hash = "sha256:a94c5a76ab46c5e6257c7972b6c8cff0574201ca7dc05647e33e795d78680563", size = 2637314, upload-time = "2026-04-21T17:05:54.5Z" }, +] + [[package]] name = "mypy-extensions" version = "1.1.0" @@ -2586,6 +2755,7 @@ dependencies = [ dev = [ { name = "build" }, { name = "coverage" }, + { name = "mypy" }, { name = "pytest" }, { name = "pytest-snapshot" }, { name = "pytest-timeout" }, @@ -2613,6 +2783,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.27.0" }, { name = "markupsafe", specifier = ">=3,<4" }, { name = "microdf-python", specifier = ">=1.0.0" }, + { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.15,<2" }, { name = "openai" }, { name = "packaging", specifier = ">=24,<27" }, { name = "policyengine", extras = ["models"], specifier = "==5.2.0" }, From a4e4c8825d4eef2e17b6bc09959f49db73d871fd Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:48:50 +0400 Subject: [PATCH 07/18] Clarify API v2 package boundaries --- docs/engineering/skills/testing.md | 5 +- docs/engineering/v2-code-organization.md | 90 ++++++++++ policyengine_api/asgi_factory.py | 4 +- policyengine_api/data/v2/metadata/__init__.py | 1 + .../dataset_read_repository.py} | 30 ++-- .../model_read_repository.py} | 12 +- .../parameter_read_repository.py} | 67 ++++---- .../parameter_tree_read_repository.py} | 89 +++++----- .../schemas.py => metadata/read_models.py} | 121 +------------- .../read_repository.py} | 12 +- .../region_read_repository.py} | 47 +++--- .../variable_read_repository.py} | 28 ++-- .../data/v2/policies/canonicalization.py | 4 +- .../{catalog.py => catalog_repository.py} | 4 +- .../v2/policies/legacy_mapping_repository.py | 69 ++++++++ .../policies/{query.py => read_repository.py} | 2 +- .../{persistence.py => write_repository.py} | 6 +- ...legacy.py => legacy_mapping_repository.py} | 158 +++++------------- .../{query.py => read_repository.py} | 2 +- .../{persistence.py => write_repository.py} | 6 +- .../fastapi_routes/dependencies.py | 20 ++- .../fastapi_routes/v2/metadata/common.py | 6 +- .../v2/metadata/geography_routes.py | 4 +- .../v2/metadata/model_routes.py | 2 +- .../v2/metadata/parameter_routes.py | 2 +- .../v2/metadata/response_models.py | 141 ++++++++++++++++ .../fastapi_routes/v2/policies/__init__.py | 1 + .../v2/policies/request_models.py | 10 ++ .../v2/policies/response_models.py} | 12 +- .../{policy_routes.py => policies/routes.py} | 16 +- policyengine_api/fastapi_routes/v2/routes.py | 8 +- .../v2/user_policies/__init__.py | 1 + .../v2/user_policies/request_models.py | 14 ++ .../v2/user_policies/response_models.py} | 16 +- .../routes.py} | 14 +- policyengine_api/services/policy_mirroring.py | 20 ++- policyengine_api/services/policy_service.py | 4 +- .../services/user_policy_mirroring.py | 24 ++- .../services/user_policy_service.py | 4 +- .../services/v2/metadata/__init__.py | 1 + .../service.py} | 34 ++-- .../services/v2/policies/__init__.py | 1 + .../v2/policies/commands.py} | 0 .../services/v2/policies/legacy_service.py | 109 ++++++++++++ .../v2/policies/legacy_translation.py} | 128 +------------- .../service.py} | 20 ++- .../services/v2/user_policies/__init__.py | 1 + .../v2/user_policies/commands.py} | 0 .../v2/user_policies/legacy_service.py | 67 ++++++++ .../v2/user_policies/legacy_translation.py | 74 ++++++++ .../service.py} | 30 ++-- pyproject.toml | 1 + .../contract/test_policy_v2_compatibility.py | 6 +- tests/contract/test_v1_route_contracts.py | 4 +- .../integration/test_v1_policy_dual_write.py | 6 +- .../test_v1_user_policy_dual_write.py | 2 +- tests/integration/test_v2_metadata_routes.py | 2 +- .../integration/test_v2_policy_persistence.py | 16 +- .../test_v2_user_policy_mirroring.py | 14 +- .../routes/test_policy_dual_write_routes.py | 4 +- .../test_user_policy_dual_write_routes.py | 8 +- tests/unit/services/test_policy_mirroring.py | 6 +- .../services/test_user_policy_mirroring.py | 8 +- .../unit/services/test_user_policy_service.py | 2 +- tests/unit/v2/test_metadata_routes.py | 6 +- tests/unit/v2/test_metadata_service.py | 64 ++++--- tests/unit/v2/test_policy_canonicalization.py | 4 +- tests/unit/v2/test_policy_catalog.py | 4 +- tests/unit/v2/test_policy_commands.py | 2 +- .../unit/v2/test_policy_legacy_translation.py | 2 +- .../v2/test_policy_persistence_statements.py | 8 +- tests/unit/v2/test_policy_query.py | 2 +- tests/unit/v2/test_policy_routes.py | 14 +- tests/unit/v2/test_user_policy_legacy.py | 13 +- tests/unit/v2/test_user_policy_routes.py | 12 +- tests/unit/v2/test_user_policy_service.py | 10 +- 76 files changed, 1067 insertions(+), 694 deletions(-) create mode 100644 docs/engineering/v2-code-organization.md create mode 100644 policyengine_api/data/v2/metadata/__init__.py rename policyengine_api/data/v2/{catalog/dataset_query.py => metadata/dataset_read_repository.py} (68%) rename policyengine_api/data/v2/{catalog/model_query.py => metadata/model_read_repository.py} (90%) rename policyengine_api/data/v2/{catalog/parameter_query.py => metadata/parameter_read_repository.py} (73%) rename policyengine_api/data/v2/{catalog/parameter_tree_query.py => metadata/parameter_tree_read_repository.py} (58%) rename policyengine_api/data/v2/{catalog/schemas.py => metadata/read_models.py} (54%) rename policyengine_api/data/v2/{catalog/query_support.py => metadata/read_repository.py} (89%) rename policyengine_api/data/v2/{catalog/region_query.py => metadata/region_read_repository.py} (77%) rename policyengine_api/data/v2/{catalog/variable_query.py => metadata/variable_read_repository.py} (72%) rename policyengine_api/data/v2/policies/{catalog.py => catalog_repository.py} (94%) create mode 100644 policyengine_api/data/v2/policies/legacy_mapping_repository.py rename policyengine_api/data/v2/policies/{query.py => read_repository.py} (98%) rename policyengine_api/data/v2/policies/{persistence.py => write_repository.py} (96%) rename policyengine_api/data/v2/user_policies/{legacy.py => legacy_mapping_repository.py} (62%) rename policyengine_api/data/v2/user_policies/{query.py => read_repository.py} (97%) rename policyengine_api/data/v2/user_policies/{persistence.py => write_repository.py} (92%) create mode 100644 policyengine_api/fastapi_routes/v2/metadata/response_models.py create mode 100644 policyengine_api/fastapi_routes/v2/policies/__init__.py create mode 100644 policyengine_api/fastapi_routes/v2/policies/request_models.py rename policyengine_api/{data/v2/policies/api_schemas.py => fastapi_routes/v2/policies/response_models.py} (89%) rename policyengine_api/fastapi_routes/v2/{policy_routes.py => policies/routes.py} (93%) create mode 100644 policyengine_api/fastapi_routes/v2/user_policies/__init__.py create mode 100644 policyengine_api/fastapi_routes/v2/user_policies/request_models.py rename policyengine_api/{data/v2/user_policies/api_schemas.py => fastapi_routes/v2/user_policies/response_models.py} (85%) rename policyengine_api/fastapi_routes/v2/{user_policy_routes.py => user_policies/routes.py} (95%) create mode 100644 policyengine_api/services/v2/metadata/__init__.py rename policyengine_api/services/v2/{metadata_service.py => metadata/service.py} (51%) create mode 100644 policyengine_api/services/v2/policies/__init__.py rename policyengine_api/{data/v2/policies/schemas.py => services/v2/policies/commands.py} (100%) create mode 100644 policyengine_api/services/v2/policies/legacy_service.py rename policyengine_api/{data/v2/policies/legacy.py => services/v2/policies/legacy_translation.py} (59%) rename policyengine_api/services/v2/{policy_service.py => policies/service.py} (87%) create mode 100644 policyengine_api/services/v2/user_policies/__init__.py rename policyengine_api/{data/v2/user_policies/schemas.py => services/v2/user_policies/commands.py} (100%) create mode 100644 policyengine_api/services/v2/user_policies/legacy_service.py create mode 100644 policyengine_api/services/v2/user_policies/legacy_translation.py rename policyengine_api/services/v2/{user_policy_service.py => user_policies/service.py} (87%) diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index dca27a278..d55fd25fc 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -159,8 +159,9 @@ that result in the handoff instead of hiding it. ## Phase 10 Policy Migration Run the configured static type check for the Phase 10 v2 query, route, -application-service, policy-persistence, and association-persistence modules. -The configured file set deliberately excludes the existing v1 implementation: +application-service, metadata-read-repository, policy-repository, and +association-repository modules. The configured file set deliberately excludes +the existing v1 implementation: ```bash uv run --frozen --extra dev mypy diff --git a/docs/engineering/v2-code-organization.md b/docs/engineering/v2-code-organization.md new file mode 100644 index 000000000..ded98785a --- /dev/null +++ b/docs/engineering/v2-code-organization.md @@ -0,0 +1,90 @@ +# API v2 Code Organization + +API v2 resource code is divided by both resource and responsibility. Public +HTTP behavior must not be implemented in database repositories, and database +sessions and transactions must not be opened by route modules. + +## HTTP adapters + +Resource-specific FastAPI code lives under `policyengine_api/fastapi_routes/v2/`: + +```text +policies/ + request_models.py + response_models.py + routes.py +user_policies/ + request_models.py + response_models.py + routes.py +metadata/ + response_models.py + common.py + *_routes.py +``` + +Request models describe HTTP request bodies. Response models describe the +public response envelope and OpenAPI output. Route modules validate HTTP-only +conditions, invoke application services, and convert typed failures to HTTP +responses. + +## Application services + +Resource-specific application code lives under `policyengine_api/services/v2/`: + +```text +policies/ + commands.py + legacy_translation.py + legacy_service.py + service.py +user_policies/ + commands.py + legacy_translation.py + legacy_service.py + service.py +metadata/ + service.py +``` + +Command models are independent of FastAPI and Flask. Native services own +request-level database sessions and transaction boundaries. Legacy translation +converts committed v1 snapshots into v2 commands. Legacy services coordinate +all work that must occur inside one Supabase transaction. + +## Database access + +SQL reads and writes live under `policyengine_api/data/v2/`: + +```text +policies/ + read_repository.py + write_repository.py + catalog_repository.py + legacy_mapping_repository.py + canonicalization.py +user_policies/ + read_repository.py + write_repository.py + legacy_mapping_repository.py +metadata/ + read_models.py + read_repository.py + *_read_repository.py +``` + +Read repositories execute selections and return framework-neutral read models. +Write repositories mutate SQLModel rows using a caller-provided session. +Legacy mapping repositories contain durable identity-mapping SQL and conflict +handling. The shared `data/v2/catalog/` package remains responsible for catalog +initialization, publication, and catalog selection used by multiple resources. + +The ordinary request direction is: + +```text +route -> service -> repository -> SQLModel tables +``` + +HTTP response models may consume framework-neutral repository read models. +Repository write functions may consume immutable application command models, +but they must not import route modules or construct HTTP responses. diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index e57bd3e28..a7700246c 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -19,12 +19,12 @@ from policyengine_api.fastapi_routes.specification import ( build_specification_router, ) -from policyengine_api.fastapi_routes.v2.policy_routes import ( +from policyengine_api.fastapi_routes.v2.policies.routes import ( PolicyRequestTooLargeError, policy_error_response, ) from policyengine_api.fastapi_routes.v2.routes import build_v2_router -from policyengine_api.fastapi_routes.v2.user_policy_routes import ( +from policyengine_api.fastapi_routes.v2.user_policies.routes import ( user_policy_error_response, ) from policyengine_api.migration_flags import ( diff --git a/policyengine_api/data/v2/metadata/__init__.py b/policyengine_api/data/v2/metadata/__init__.py new file mode 100644 index 000000000..b92e3f7de --- /dev/null +++ b/policyengine_api/data/v2/metadata/__init__.py @@ -0,0 +1 @@ +"""Read models and database repositories for API v2 metadata.""" diff --git a/policyengine_api/data/v2/catalog/dataset_query.py b/policyengine_api/data/v2/metadata/dataset_read_repository.py similarity index 68% rename from policyengine_api/data/v2/catalog/dataset_query.py rename to policyengine_api/data/v2/metadata/dataset_read_repository.py index 755d5b006..f0bf9e337 100644 --- a/policyengine_api/data/v2/catalog/dataset_query.py +++ b/policyengine_api/data/v2/metadata/dataset_read_repository.py @@ -1,17 +1,17 @@ -"""Logical input-dataset metadata queries.""" +"""Logical input-dataset metadata database reads.""" from __future__ import annotations from uuid import UUID -from sqlmodel import select -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, +from sqlmodel import col, select +from policyengine_api.data.v2.metadata.read_repository import ( + MetadataReadRepositoryBase, MetadataResourceNotFoundError, page_result, query_rows, ) -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.data.v2.metadata.read_models import ( MetadataDataset, MetadataDetailResult, MetadataPageResult, @@ -28,8 +28,8 @@ def _dataset(dataset: Dataset) -> MetadataDataset: ) -class DatasetQueryMethods(MetadataQueryContext): - """Route-facing logical input-dataset query methods.""" +class DatasetReadRepository(MetadataReadRepositoryBase): + """Read logical input datasets from the selected catalog.""" def list_datasets( self, @@ -49,11 +49,11 @@ def list_datasets( self._session, select(Dataset) .where( - Dataset.tax_benefit_model_version_id == selected.model_version.id, - Dataset.is_output_dataset.is_(False), - Dataset.storage_path.is_(None), + col(Dataset.tax_benefit_model_version_id) == selected.model_version.id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), ) - .order_by(Dataset.name) + .order_by(col(Dataset.name)) .offset(offset) .limit(limit + 1), ) @@ -74,10 +74,10 @@ def get_dataset( rows = query_rows( self._session, select(Dataset).where( - Dataset.id == dataset_id, - Dataset.tax_benefit_model_version_id == selected.model_version.id, - Dataset.is_output_dataset.is_(False), - Dataset.storage_path.is_(None), + col(Dataset.id) == dataset_id, + col(Dataset.tax_benefit_model_version_id) == selected.model_version.id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), ), ) if not rows: diff --git a/policyengine_api/data/v2/catalog/model_query.py b/policyengine_api/data/v2/metadata/model_read_repository.py similarity index 90% rename from policyengine_api/data/v2/catalog/model_query.py rename to policyengine_api/data/v2/metadata/model_read_repository.py index 17a0e130b..a4f6855d5 100644 --- a/policyengine_api/data/v2/catalog/model_query.py +++ b/policyengine_api/data/v2/metadata/model_read_repository.py @@ -1,16 +1,16 @@ -"""Tax-benefit model and model-version metadata queries.""" +"""Tax-benefit model and model-version metadata database reads.""" from __future__ import annotations from uuid import UUID from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, +from policyengine_api.data.v2.metadata.read_repository import ( + MetadataReadRepositoryBase, MetadataResourceNotFoundError, page_result, ) -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.data.v2.metadata.read_models import ( MetadataDetailResult, MetadataModel, MetadataModelSelectionResult, @@ -38,8 +38,8 @@ def _model_version(selected: SelectedCatalog) -> MetadataModelVersionDetail: ) -class ModelQueryMethods(MetadataQueryContext): - """Route-facing model and model-version query methods.""" +class ModelReadRepository(MetadataReadRepositoryBase): + """Read tax-benefit models and model versions from the selected catalog.""" def list_models( self, diff --git a/policyengine_api/data/v2/catalog/parameter_query.py b/policyengine_api/data/v2/metadata/parameter_read_repository.py similarity index 73% rename from policyengine_api/data/v2/catalog/parameter_query.py rename to policyengine_api/data/v2/metadata/parameter_read_repository.py index c3807f1f2..90367bb03 100644 --- a/policyengine_api/data/v2/catalog/parameter_query.py +++ b/policyengine_api/data/v2/metadata/parameter_read_repository.py @@ -1,4 +1,4 @@ -"""Parameter, parameter-tree, and canonical parameter-value queries.""" +"""Parameter and canonical parameter-value metadata database reads.""" from __future__ import annotations @@ -6,19 +6,19 @@ from uuid import UUID import sqlalchemy as sa -from sqlmodel import select -from policyengine_api.data.v2.catalog.parameter_tree_query import ( +from sqlmodel import col, select +from policyengine_api.data.v2.metadata.parameter_tree_read_repository import ( parameter_children_from_rows, parameter_children_query, ) -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, +from policyengine_api.data.v2.metadata.read_repository import ( + MetadataReadRepositoryBase, MetadataResourceNotFoundError, escape_like, page_result, query_rows, ) -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.data.v2.metadata.read_models import ( MetadataCanonicalParameterValue, MetadataDetailResult, MetadataPageResult, @@ -60,8 +60,8 @@ def _utc_day_start(selected_time: datetime) -> datetime: ) -class ParameterQueryMethods(MetadataQueryContext): - """Route-facing parameter, tree, and canonical-value query methods.""" +class ParameterReadRepository(MetadataReadRepositoryBase): + """Read parameters and canonical values from the selected catalog.""" def list_parameters( self, @@ -79,20 +79,20 @@ def list_parameters( limit=limit, ) statement = select(Parameter).where( - Parameter.tax_benefit_model_version_id == selected.model_version.id + col(Parameter.tax_benefit_model_version_id) == selected.model_version.id ) if search: pattern = f"%{escape_like(search)}%" statement = statement.where( sa.or_( - Parameter.name.ilike(pattern, escape="\\"), - Parameter.label.ilike(pattern, escape="\\"), - Parameter.description.ilike(pattern, escape="\\"), + col(Parameter.name).ilike(pattern, escape="\\"), + col(Parameter.label).ilike(pattern, escape="\\"), + col(Parameter.description).ilike(pattern, escape="\\"), ) ) rows = query_rows( self._session, - statement.order_by(Parameter.name).offset(offset).limit(limit + 1), + statement.order_by(col(Parameter.name)).offset(offset).limit(limit + 1), ) return page_result( selected, @@ -111,8 +111,9 @@ def get_parameter( rows = query_rows( self._session, select(Parameter).where( - Parameter.id == parameter_id, - Parameter.tax_benefit_model_version_id == selected.model_version.id, + col(Parameter.id) == parameter_id, + col(Parameter.tax_benefit_model_version_id) + == selected.model_version.id, ), ) if not rows: @@ -175,30 +176,33 @@ def list_parameter_values( ) statement = ( select(ParameterValue) - .join(Parameter, Parameter.id == ParameterValue.parameter_id) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) .where( - Parameter.tax_benefit_model_version_id == selected.model_version.id, - ParameterValue.policy_id.is_(None), - ParameterValue.dynamic_id.is_(None), + col(Parameter.tax_benefit_model_version_id) + == selected.model_version.id, + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), ) ) if parameter_id is not None: - statement = statement.where(ParameterValue.parameter_id == parameter_id) + statement = statement.where( + col(ParameterValue.parameter_id) == parameter_id + ) if current: selected_day = _utc_day_start(now or datetime.now(timezone.utc)) statement = statement.where( - ParameterValue.start_date <= selected_day, + col(ParameterValue.start_date) <= selected_day, sa.or_( - ParameterValue.end_date.is_(None), - ParameterValue.end_date >= selected_day, + col(ParameterValue.end_date).is_(None), + col(ParameterValue.end_date) >= selected_day, ), ) rows = query_rows( self._session, statement.order_by( - Parameter.name, - ParameterValue.start_date.desc(), - ParameterValue.id, + col(Parameter.name), + col(ParameterValue.start_date).desc(), + col(ParameterValue.id), ) .offset(offset) .limit(limit + 1), @@ -220,12 +224,13 @@ def get_parameter_value( rows = query_rows( self._session, select(ParameterValue) - .join(Parameter, Parameter.id == ParameterValue.parameter_id) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) .where( - ParameterValue.id == value_id, - Parameter.tax_benefit_model_version_id == selected.model_version.id, - ParameterValue.policy_id.is_(None), - ParameterValue.dynamic_id.is_(None), + col(ParameterValue.id) == value_id, + col(Parameter.tax_benefit_model_version_id) + == selected.model_version.id, + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), ), ) if not rows: diff --git a/policyengine_api/data/v2/catalog/parameter_tree_query.py b/policyengine_api/data/v2/metadata/parameter_tree_read_repository.py similarity index 58% rename from policyengine_api/data/v2/catalog/parameter_tree_query.py rename to policyengine_api/data/v2/metadata/parameter_tree_read_repository.py index cc4c856a7..238ae92fa 100644 --- a/policyengine_api/data/v2/catalog/parameter_tree_query.py +++ b/policyengine_api/data/v2/metadata/parameter_tree_read_repository.py @@ -1,13 +1,14 @@ -"""SQL query and row conversion for direct parameter-tree children.""" +"""Database reads for direct parameter-tree children.""" from __future__ import annotations +from typing import Any from uuid import UUID import sqlalchemy as sa -from sqlmodel import select +from sqlmodel import col -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.data.v2.metadata.read_models import ( MetadataParameterChild, MetadataParameterSummary, ) @@ -18,7 +19,7 @@ def _escaped_like(value: str) -> str: return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") -def _path_segment(remainder: object, dialect: str) -> object: +def _path_segment(remainder: Any, dialect: str) -> Any: dot_position = ( sa.func.instr(remainder, ".") if dialect == "sqlite" @@ -30,21 +31,21 @@ def _path_segment(remainder: object, dialect: str) -> object: ) -def _child_path(column: object, prefix: str, dialect: str) -> object: +def _child_path(column: Any, prefix: str, dialect: str) -> Any: remainder = sa.func.substr(column, len(prefix) + 1) return sa.literal(prefix) + _path_segment(remainder, dialect) def _direct_child_path( - column: object, - parent_path: object, + column: Any, + parent_path: Any, dialect: str, -) -> object: +) -> Any: remainder = sa.func.substr(column, sa.func.length(parent_path) + 2) return parent_path + "." + _path_segment(remainder, dialect) -def _has_path_prefix(column: object, parent_path: object) -> object: +def _has_path_prefix(column: Any, parent_path: Any) -> Any: return ( sa.func.substr(column, 1, sa.func.length(parent_path) + 1) == parent_path + "." ) @@ -57,80 +58,88 @@ def parameter_children_query( dialect: str, offset: int, limit: int, -) -> object: +) -> Any: """Build one bounded query for a parameter path's direct children.""" prefix = f"{parent_path}." if parent_path else "" escaped_prefix = _escaped_like(prefix) - node_child_path = _child_path(ParameterNode.name, prefix, dialect) - parameter_child_path = _child_path(Parameter.name, prefix, dialect) + node_name = col(ParameterNode.name) + node_model_version_id = col(ParameterNode.tax_benefit_model_version_id) + parameter_name = col(Parameter.name) + parameter_model_version_id = col(Parameter.tax_benefit_model_version_id) + node_child_path = _child_path(node_name, prefix, dialect) + parameter_child_path = _child_path(parameter_name, prefix, dialect) paths = sa.union( - select(node_child_path.label("path")).where( - ParameterNode.tax_benefit_model_version_id == model_version_id, - ParameterNode.name.like(f"{escaped_prefix}%", escape="\\"), + sa.select(node_child_path.label("path")).where( + node_model_version_id == model_version_id, + node_name.like(f"{escaped_prefix}%", escape="\\"), ), - select(parameter_child_path.label("path")).where( - Parameter.tax_benefit_model_version_id == model_version_id, - Parameter.name.like(f"{escaped_prefix}%", escape="\\"), + sa.select(parameter_child_path.label("path")).where( + parameter_model_version_id == model_version_id, + parameter_name.like(f"{escaped_prefix}%", escape="\\"), ), ).subquery() direct_child_paths = sa.union( - select( + sa.select( _direct_child_path( - ParameterNode.name, + node_name, paths.c.path, dialect, ).label("path") ) .where( - ParameterNode.tax_benefit_model_version_id == model_version_id, - _has_path_prefix(ParameterNode.name, paths.c.path), + node_model_version_id == model_version_id, + _has_path_prefix(node_name, paths.c.path), ) .correlate(paths), - select( + sa.select( _direct_child_path( - Parameter.name, + parameter_name, paths.c.path, dialect, ).label("path") ) .where( - Parameter.tax_benefit_model_version_id == model_version_id, - _has_path_prefix(Parameter.name, paths.c.path), + parameter_model_version_id == model_version_id, + _has_path_prefix(parameter_name, paths.c.path), ) .correlate(paths), ).subquery() direct_child_count = ( - select(sa.func.count()) + sa.select(sa.func.count()) .select_from(direct_child_paths) .correlate(paths) .scalar_subquery() ) - is_node = sa.or_(direct_child_count > 0, Parameter.id.is_(None)) + parameter_id = col(Parameter.id) + is_node = sa.or_(direct_child_count > 0, parameter_id.is_(None)) return ( - select( + sa.select( paths.c.path, - sa.func.coalesce(ParameterNode.label, Parameter.label).label("label"), + sa.func.coalesce( + col(ParameterNode.label), + col(Parameter.label), + ).label("label"), sa.case((is_node, "node"), else_="parameter").label("type"), sa.case((is_node, direct_child_count), else_=None).label("child_count"), - Parameter.id.label("parameter_id"), - Parameter.label.label("parameter_label"), - Parameter.description.label("parameter_description"), - Parameter.data_type.label("parameter_data_type"), - Parameter.unit.label("parameter_unit"), + parameter_id.label("parameter_id"), + col(Parameter.label).label("parameter_label"), + col(Parameter.description).label("parameter_description"), + col(Parameter.data_type).label("parameter_data_type"), + col(Parameter.unit).label("parameter_unit"), ) .select_from( paths.outerjoin( ParameterNode, sa.and_( - ParameterNode.name == paths.c.path, - ParameterNode.tax_benefit_model_version_id == model_version_id, + node_name == paths.c.path, + node_model_version_id == model_version_id, ), ).outerjoin( Parameter, sa.and_( - Parameter.name == paths.c.path, - Parameter.tax_benefit_model_version_id == model_version_id, + parameter_name == paths.c.path, + parameter_model_version_id == model_version_id, ), ) ) @@ -140,7 +149,7 @@ def parameter_children_query( ) -def parameter_children_from_rows(rows: list) -> list[MetadataParameterChild]: +def parameter_children_from_rows(rows: list[Any]) -> list[MetadataParameterChild]: """Convert parameter-tree query rows into typed direct-child records.""" items = [] diff --git a/policyengine_api/data/v2/catalog/schemas.py b/policyengine_api/data/v2/metadata/read_models.py similarity index 54% rename from policyengine_api/data/v2/catalog/schemas.py rename to policyengine_api/data/v2/metadata/read_models.py index ddd01fb8e..273319215 100644 --- a/policyengine_api/data/v2/catalog/schemas.py +++ b/policyengine_api/data/v2/metadata/read_models.py @@ -1,13 +1,13 @@ -"""Typed response schemas for the dormant v2 metadata preview.""" +"""Framework-neutral read models for API v2 metadata resources.""" from __future__ import annotations from datetime import datetime from enum import StrEnum -from typing import Annotated, Generic, Literal, TypeVar +from typing import Generic, Literal, TypeVar from uuid import UUID -from pydantic import BaseModel, ConfigDict, JsonValue, StringConstraints +from pydantic import BaseModel, ConfigDict, JsonValue class StrictResponseModel(BaseModel): @@ -150,118 +150,3 @@ class MetadataEconomyOptionsResult(StrictResponseModel): region: list[MetadataRegionOption] time_period: list[MetadataTimePeriodOption] datasets: list[MetadataDatasetOption] - - -class MetadataResourceSuccessResponse(StrictResponseModel, Generic[ResourceT]): - status: Literal["ok"] = "ok" - message: None = None - result: ResourceT - - -class MetadataModelPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataModel]] -): - pass - - -class MetadataModelDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataModel]] -): - pass - - -class MetadataModelSelectionResponse( - MetadataResourceSuccessResponse[MetadataModelSelectionResult] -): - pass - - -class MetadataModelVersionPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataModelVersionDetail]] -): - pass - - -class MetadataModelVersionDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataModelVersionDetail]] -): - pass - - -class MetadataVariablePageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataVariable]] -): - pass - - -class MetadataVariableDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataVariable]] -): - pass - - -class MetadataParameterPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataParameterSummary]] -): - pass - - -class MetadataParameterDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataParameterSummary]] -): - pass - - -class MetadataParameterChildPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataParameterChild]] -): - pass - - -class MetadataParameterValuePageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataCanonicalParameterValue]] -): - pass - - -class MetadataParameterValueDetailResponse( - MetadataResourceSuccessResponse[ - MetadataDetailResult[MetadataCanonicalParameterValue] - ] -): - pass - - -class MetadataDatasetPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataDataset]] -): - pass - - -class MetadataDatasetDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataDataset]] -): - pass - - -class MetadataRegionPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataRegion]] -): - pass - - -class MetadataRegionDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataRegion]] -): - pass - - -class MetadataEconomyOptionsResponse( - MetadataResourceSuccessResponse[MetadataEconomyOptionsResult] -): - pass - - -class MetadataErrorResponse(StrictResponseModel): - status: Literal["error"] = "error" - message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] diff --git a/policyengine_api/data/v2/catalog/query_support.py b/policyengine_api/data/v2/metadata/read_repository.py similarity index 89% rename from policyengine_api/data/v2/catalog/query_support.py rename to policyengine_api/data/v2/metadata/read_repository.py index 8dd0b49e5..3bc424d24 100644 --- a/policyengine_api/data/v2/catalog/query_support.py +++ b/policyengine_api/data/v2/metadata/read_repository.py @@ -1,8 +1,8 @@ -"""Shared execution and pagination for v2 metadata resource queries.""" +"""Shared database execution and pagination for v2 metadata reads.""" from __future__ import annotations -from typing import TypeVar +from typing import Any, TypeVar from sqlalchemy.exc import SQLAlchemyError from sqlmodel import Session @@ -13,7 +13,7 @@ select_catalog as select_metadata_catalog, validate_policyengine_version, ) -from policyengine_api.data.v2.catalog.schemas import MetadataPageResult +from policyengine_api.data.v2.metadata.read_models import MetadataPageResult class MetadataResourceNotFoundError(LookupError): @@ -27,8 +27,8 @@ class InvalidMetadataPageError(ValueError): ResourceT = TypeVar("ResourceT") -class MetadataQueryContext: - """Own the session and exact catalog selection shared by resource queries.""" +class MetadataReadRepositoryBase: + """Own the session and catalog selection shared by metadata repositories.""" def __init__(self, session: Session, *, running_policyengine_version: str): self._session = session @@ -95,7 +95,7 @@ def validate_metadata_page(offset: int, limit: int) -> tuple[int, int]: return offset, limit -def query_rows(session: Session, statement: object) -> list: +def query_rows(session: Session, statement: Any) -> list[Any]: """Execute one read statement and translate database failures.""" try: diff --git a/policyengine_api/data/v2/catalog/region_query.py b/policyengine_api/data/v2/metadata/region_read_repository.py similarity index 77% rename from policyengine_api/data/v2/catalog/region_query.py rename to policyengine_api/data/v2/metadata/region_read_repository.py index 8ac0f3417..2f574f915 100644 --- a/policyengine_api/data/v2/catalog/region_query.py +++ b/policyengine_api/data/v2/metadata/region_read_repository.py @@ -1,28 +1,29 @@ -"""Region and economy-option metadata queries.""" +"""Region and economy-option metadata database reads.""" from __future__ import annotations from uuid import UUID -from sqlmodel import select +from sqlmodel import col, select from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.data.v2.catalog.catalog_selection import ( MetadataCatalogUnavailableError, ) -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, +from policyengine_api.data.v2.metadata.read_repository import ( + MetadataReadRepositoryBase, MetadataResourceNotFoundError, page_result, query_rows, ) -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.data.v2.metadata.read_models import ( MetadataDatasetOption, MetadataDetailResult, MetadataEconomyOptionsResult, MetadataPageResult, MetadataRegion, MetadataRegionOption, + MetadataRegionType, MetadataTimePeriodOption, ) from policyengine_api.data.v2.models import Dataset, Region @@ -33,7 +34,7 @@ def _region(region: Region) -> MetadataRegion: id=region.id, code=region.code, label=region.label, - region_type=region.region_type.value, + region_type=MetadataRegionType(region.region_type.value), requires_filter=region.requires_filter, filter_field=region.filter_field, filter_value=region.filter_value, @@ -45,8 +46,8 @@ def _region(region: Region) -> MetadataRegion: ) -class RegionQueryMethods(MetadataQueryContext): - """Route-facing region and economy-option query methods.""" +class RegionReadRepository(MetadataReadRepositoryBase): + """Read regions and economy options from the selected catalog.""" def list_regions( self, @@ -64,13 +65,13 @@ def list_regions( limit=limit, ) statement = select(Region).where( - Region.tax_benefit_model_version_id == selected.model_version.id + col(Region.tax_benefit_model_version_id) == selected.model_version.id ) if region_type is not None: - statement = statement.where(Region.region_type == region_type) + statement = statement.where(col(Region.region_type) == region_type) rows = query_rows( self._session, - statement.order_by(Region.code).offset(offset).limit(limit + 1), + statement.order_by(col(Region.code)).offset(offset).limit(limit + 1), ) return page_result( selected, @@ -89,8 +90,8 @@ def get_region( rows = query_rows( self._session, select(Region).where( - Region.id == region_id, - Region.tax_benefit_model_version_id == selected.model_version.id, + col(Region.id) == region_id, + col(Region.tax_benefit_model_version_id) == selected.model_version.id, ), ) if not rows: @@ -110,8 +111,8 @@ def get_region_by_code( rows = query_rows( self._session, select(Region).where( - Region.code == region_code, - Region.tax_benefit_model_version_id == selected.model_version.id, + col(Region.code) == region_code, + col(Region.tax_benefit_model_version_id) == selected.model_version.id, ), ) if not rows: @@ -130,8 +131,10 @@ def get_economy_options( regions = query_rows( self._session, select(Region) - .where(Region.tax_benefit_model_version_id == selected.model_version.id) - .order_by(Region.code), + .where( + col(Region.tax_benefit_model_version_id) == selected.model_version.id + ) + .order_by(col(Region.code)), ) national_region = next( (region for region in regions if region.code == country_id), @@ -144,10 +147,10 @@ def get_economy_options( datasets = query_rows( self._session, select(Dataset).where( - Dataset.id == national_region.default_dataset_id, - Dataset.tax_benefit_model_version_id == selected.model_version.id, - Dataset.is_output_dataset.is_(False), - Dataset.storage_path.is_(None), + col(Dataset.id) == national_region.default_dataset_id, + col(Dataset.tax_benefit_model_version_id) == selected.model_version.id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), ), ) if len(datasets) != 1: @@ -172,7 +175,7 @@ def get_economy_options( MetadataRegionOption( name=region.code, label=region.label, - type=region.region_type.value, + type=MetadataRegionType(region.region_type.value), ) for region in regions ], diff --git a/policyengine_api/data/v2/catalog/variable_query.py b/policyengine_api/data/v2/metadata/variable_read_repository.py similarity index 72% rename from policyengine_api/data/v2/catalog/variable_query.py rename to policyengine_api/data/v2/metadata/variable_read_repository.py index 6f6c79dd9..b4f51f659 100644 --- a/policyengine_api/data/v2/catalog/variable_query.py +++ b/policyengine_api/data/v2/metadata/variable_read_repository.py @@ -1,19 +1,19 @@ -"""Variable metadata queries.""" +"""Variable metadata database reads.""" from __future__ import annotations from uuid import UUID import sqlalchemy as sa -from sqlmodel import select -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, +from sqlmodel import col, select +from policyengine_api.data.v2.metadata.read_repository import ( + MetadataReadRepositoryBase, MetadataResourceNotFoundError, escape_like, page_result, query_rows, ) -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.data.v2.metadata.read_models import ( MetadataDetailResult, MetadataPageResult, MetadataVariable, @@ -36,8 +36,8 @@ def _variable(variable: Variable) -> MetadataVariable: ) -class VariableQueryMethods(MetadataQueryContext): - """Route-facing variable query methods.""" +class VariableReadRepository(MetadataReadRepositoryBase): + """Read variables from the selected catalog.""" def list_variables( self, @@ -55,20 +55,20 @@ def list_variables( limit=limit, ) statement = select(Variable).where( - Variable.tax_benefit_model_version_id == selected.model_version.id + col(Variable.tax_benefit_model_version_id) == selected.model_version.id ) if search: pattern = f"%{escape_like(search)}%" statement = statement.where( sa.or_( - Variable.name.ilike(pattern, escape="\\"), - Variable.label.ilike(pattern, escape="\\"), - Variable.description.ilike(pattern, escape="\\"), + col(Variable.name).ilike(pattern, escape="\\"), + col(Variable.label).ilike(pattern, escape="\\"), + col(Variable.description).ilike(pattern, escape="\\"), ) ) rows = query_rows( self._session, - statement.order_by(Variable.name).offset(offset).limit(limit + 1), + statement.order_by(col(Variable.name)).offset(offset).limit(limit + 1), ) return page_result( selected, @@ -87,8 +87,8 @@ def get_variable( rows = query_rows( self._session, select(Variable).where( - Variable.id == variable_id, - Variable.tax_benefit_model_version_id == selected.model_version.id, + col(Variable.id) == variable_id, + col(Variable.tax_benefit_model_version_id) == selected.model_version.id, ), ) if not rows: diff --git a/policyengine_api/data/v2/policies/canonicalization.py b/policyengine_api/data/v2/policies/canonicalization.py index 9b1b0df9e..524a5f2ec 100644 --- a/policyengine_api/data/v2/policies/canonicalization.py +++ b/policyengine_api/data/v2/policies/canonicalization.py @@ -9,7 +9,9 @@ import json from typing import Any -from policyengine_api.data.v2.policies.schemas import ResolvedPolicyCreateCommand +from policyengine_api.services.v2.policies.commands import ( + ResolvedPolicyCreateCommand, +) POLICY_CANONICALIZATION_VERSION = 1 diff --git a/policyengine_api/data/v2/policies/catalog.py b/policyengine_api/data/v2/policies/catalog_repository.py similarity index 94% rename from policyengine_api/data/v2/policies/catalog.py rename to policyengine_api/data/v2/policies/catalog_repository.py index c73bfcf64..f99aece8e 100644 --- a/policyengine_api/data/v2/policies/catalog.py +++ b/policyengine_api/data/v2/policies/catalog_repository.py @@ -1,4 +1,4 @@ -"""Catalog binding for immutable v2 policy commands.""" +"""Catalog database validation for immutable v2 policy commands.""" from __future__ import annotations @@ -9,7 +9,7 @@ from policyengine_api.constants import POLICYENGINE_VERSION from policyengine_api.data.v2.catalog.catalog_selection import select_catalog from policyengine_api.data.v2.models import Parameter -from policyengine_api.data.v2.policies.schemas import ( +from policyengine_api.services.v2.policies.commands import ( PolicyCreateCommand, ResolvedPolicyCreateCommand, ) diff --git a/policyengine_api/data/v2/policies/legacy_mapping_repository.py b/policyengine_api/data/v2/policies/legacy_mapping_repository.py new file mode 100644 index 000000000..3f1ec3f4a --- /dev/null +++ b/policyengine_api/data/v2/policies/legacy_mapping_repository.py @@ -0,0 +1,69 @@ +"""Database operations for durable v1-policy-to-v2-policy mappings.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import LegacyPolicyMapping + + +class LegacyPolicyMappingIntegrityError(RuntimeError): + """Raised when one immutable v1 identity maps inconsistently.""" + + +def find_legacy_policy_mapping( + session: Session, + *, + country_id: str, + legacy_policy_id: int, + lock: bool, +) -> LegacyPolicyMapping | None: + statement = select(LegacyPolicyMapping).where( + LegacyPolicyMapping.country_id == country_id, + LegacyPolicyMapping.legacy_policy_id == legacy_policy_id, + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def verify_legacy_policy_mapping( + mapping: LegacyPolicyMapping, + *, + source_policy_hash: str, + expected_policy_id: UUID | None = None, +) -> None: + if mapping.source_policy_hash != source_policy_hash: + raise LegacyPolicyMappingIntegrityError( + "legacy policy identity was presented with a different source hash" + ) + if expected_policy_id is not None and mapping.policy_id != expected_policy_id: + raise LegacyPolicyMappingIntegrityError( + "legacy policy mapping does not match translated immutable content" + ) + + +def insert_legacy_policy_mapping( + session: Session, + *, + country_id: str, + legacy_policy_id: int, + source_policy_hash: str, + policy_id: UUID, +) -> UUID | None: + """Insert one mapping and return its UUID, or none after a conflict.""" + + return session.execute( + insert(LegacyPolicyMapping) + .values( + country_id=country_id, + legacy_policy_id=legacy_policy_id, + policy_id=policy_id, + source_policy_hash=source_policy_hash, + ) + .on_conflict_do_nothing(constraint="uq_legacy_policy_mappings_country_legacy") + .returning(col(LegacyPolicyMapping.id)) + ).scalar_one_or_none() diff --git a/policyengine_api/data/v2/policies/query.py b/policyengine_api/data/v2/policies/read_repository.py similarity index 98% rename from policyengine_api/data/v2/policies/query.py rename to policyengine_api/data/v2/policies/read_repository.py index 59cc319df..adb2cfab0 100644 --- a/policyengine_api/data/v2/policies/query.py +++ b/policyengine_api/data/v2/policies/read_repository.py @@ -1,4 +1,4 @@ -"""Country-scoped immutable v2 policy read operations.""" +"""Country-scoped database reads for immutable v2 policies.""" from __future__ import annotations diff --git a/policyengine_api/data/v2/policies/persistence.py b/policyengine_api/data/v2/policies/write_repository.py similarity index 96% rename from policyengine_api/data/v2/policies/persistence.py rename to policyengine_api/data/v2/policies/write_repository.py index 3f7ca60b7..b839f8b83 100644 --- a/policyengine_api/data/v2/policies/persistence.py +++ b/policyengine_api/data/v2/policies/write_repository.py @@ -1,4 +1,4 @@ -"""Conflict-aware PostgreSQL persistence for immutable v2 policies.""" +"""Conflict-aware PostgreSQL writes for immutable v2 policies.""" from __future__ import annotations @@ -19,7 +19,9 @@ canonical_policy_document, canonicalize_policy, ) -from policyengine_api.data.v2.policies.schemas import ResolvedPolicyCreateCommand +from policyengine_api.services.v2.policies.commands import ( + ResolvedPolicyCreateCommand, +) class PolicyPersistenceIntegrityError(RuntimeError): diff --git a/policyengine_api/data/v2/user_policies/legacy.py b/policyengine_api/data/v2/user_policies/legacy_mapping_repository.py similarity index 62% rename from policyengine_api/data/v2/user_policies/legacy.py rename to policyengine_api/data/v2/user_policies/legacy_mapping_repository.py index 9ba9c7c6c..df0433838 100644 --- a/policyengine_api/data/v2/user_policies/legacy.py +++ b/policyengine_api/data/v2/user_policies/legacy_mapping_repository.py @@ -1,14 +1,10 @@ -"""Projection and durable mapping of committed v1 saved-policy rows.""" +"""Database operations for legacy users and saved-policy association mappings.""" from __future__ import annotations from dataclasses import dataclass -from hashlib import sha256 -import json -from typing import Annotated from uuid import UUID -from pydantic import Field from sqlalchemy.dialects.postgresql import insert from sqlmodel import Session, col, select @@ -19,14 +15,9 @@ UserPolicy, ) from policyengine_api.data.v2.models.base import utc_now -from policyengine_api.data.v2.policies.legacy import ( - LegacyPolicySnapshot, - persist_legacy_policy, +from policyengine_api.services.v2.user_policies.commands import ( + UserPolicyCreateCommand, ) -from policyengine_api.data.v2.policies.schemas import StrictPolicyCommand -from policyengine_api.data.v2.user_policies.schemas import UserPolicyCreateCommand -from policyengine_api.query_parameters import CountryId, LegacyUserId - USER_POLICY_FINGERPRINT_VERSION = 1 @@ -35,27 +26,6 @@ class LegacyUserPolicyIntegrityError(RuntimeError): """Raised when source, policy, association, or mapping identity conflicts.""" -class LegacyUserPolicySnapshot(StrictPolicyCommand): - """Detached complete committed v1 saved-policy row.""" - - country_id: CountryId - legacy_user_policy_id: Annotated[int, Field(ge=0)] - reform_id: Annotated[int, Field(ge=0)] - reform_label: Annotated[str, Field(max_length=255)] | None = None - baseline_id: Annotated[int, Field(ge=0)] - baseline_label: Annotated[str, Field(max_length=255)] | None = None - user_id: LegacyUserId - year: Annotated[str, Field(max_length=32)] - geography: Annotated[str, Field(max_length=255)] - dataset: Annotated[str, Field(max_length=255)] | None = None - number_of_provisions: Annotated[int, Field(ge=0)] - api_version: Annotated[str, Field(max_length=32)] - added_date: int - updated_date: int - budgetary_impact: Annotated[str, Field(max_length=255)] | None = None - type: Annotated[str, Field(max_length=255)] | None = None - - @dataclass(frozen=True) class LegacyUserPolicyPersistenceResult: association_id: UUID @@ -65,40 +35,6 @@ class LegacyUserPolicyPersistenceResult: mapping_created: bool -def fingerprint_legacy_user_policy(snapshot: LegacyUserPolicySnapshot) -> str: - """Hash every committed source field through deterministic JSON.""" - - document = { - "fingerprint_version": USER_POLICY_FINGERPRINT_VERSION, - **snapshot.model_dump(mode="json"), - } - encoded = json.dumps( - document, - ensure_ascii=True, - allow_nan=False, - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") - return sha256(encoded).hexdigest() - - -def project_legacy_user_policy( - snapshot: LegacyUserPolicySnapshot, - *, - user_id: UUID, - policy_id: UUID, -) -> UserPolicyCreateCommand: - """Map v1 presentation data onto an association, never core policy content.""" - - return UserPolicyCreateCommand( - country_id=snapshot.country_id, - user_id=user_id, - policy_id=policy_id, - name=snapshot.reform_label, - description=None, - ) - - def _legacy_user_mapping( session: Session, legacy_user_id: str, @@ -156,13 +92,14 @@ def resolve_legacy_user_id( def _mapping( session: Session, - snapshot: LegacyUserPolicySnapshot, *, + country_id: str, + legacy_user_policy_id: int, lock: bool, ) -> LegacyUserPolicyMapping | None: statement = select(LegacyUserPolicyMapping).where( - LegacyUserPolicyMapping.country_id == snapshot.country_id, - LegacyUserPolicyMapping.legacy_user_policy_id == snapshot.legacy_user_policy_id, + LegacyUserPolicyMapping.country_id == country_id, + LegacyUserPolicyMapping.legacy_user_policy_id == legacy_user_policy_id, ) if lock: statement = statement.with_for_update() @@ -186,11 +123,12 @@ def _mapped_association( return association -def _apply_existing_mapping( +def apply_existing_legacy_user_policy_mapping( session: Session, *, mapping: LegacyUserPolicyMapping, - snapshot: LegacyUserPolicySnapshot, + country_id: str, + reform_label: str | None, fingerprint: str, user_id: UUID, policy_id: UUID, @@ -200,7 +138,7 @@ def _apply_existing_mapping( association = _mapped_association(session, mapping) if ( association.policy_id != policy_id - or association.country_id != snapshot.country_id + or association.country_id != country_id or association.user_id != user_id ): raise LegacyUserPolicyIntegrityError( @@ -236,10 +174,10 @@ def _apply_existing_mapping( ) association_updated = ( - "reform_label" in changed_fields and association.name != snapshot.reform_label + "reform_label" in changed_fields and association.name != reform_label ) if association_updated: - association.name = snapshot.reform_label + association.name = reform_label association.updated_at = utc_now() session.add(association) mapping.fingerprint_sha256 = fingerprint @@ -256,60 +194,46 @@ def _apply_existing_mapping( ) -def persist_legacy_user_policy( +def persist_legacy_user_policy_mapping( session: Session, - snapshot: LegacyUserPolicySnapshot, - reform_snapshot: LegacyPolicySnapshot, *, + country_id: str, + legacy_user_policy_id: int, + reform_label: str | None, + projection: UserPolicyCreateCommand, + fingerprint: str, source_revision: int, - changed_fields: frozenset[str] = frozenset(), + changed_fields: frozenset[str], ) -> LegacyUserPolicyPersistenceResult: - """Ensure reform, association, and both mappings in the caller transaction.""" + """Create or advance one v1 saved-policy association mapping.""" - if source_revision <= 0: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy source revision must be positive" - ) - if ( - snapshot.country_id != reform_snapshot.country_id - or snapshot.reform_id != reform_snapshot.legacy_policy_id - ): - raise LegacyUserPolicyIntegrityError( - "saved policy does not reference the supplied reform snapshot" - ) - policy_result = persist_legacy_policy(session, reform_snapshot) - user_id = resolve_legacy_user_id( + existing = _mapping( session, - legacy_user_id=snapshot.user_id, - primary_country=snapshot.country_id, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + lock=True, ) - fingerprint = fingerprint_legacy_user_policy(snapshot) - existing = _mapping(session, snapshot, lock=True) if existing is not None: - return _apply_existing_mapping( + return apply_existing_legacy_user_policy_mapping( session, mapping=existing, - snapshot=snapshot, + country_id=country_id, + reform_label=reform_label, fingerprint=fingerprint, - user_id=user_id, - policy_id=policy_result.policy_id, + user_id=projection.user_id, + policy_id=projection.policy_id, changed_fields=changed_fields, source_revision=source_revision, ) - projection = project_legacy_user_policy( - snapshot, - user_id=user_id, - policy_id=policy_result.policy_id, - ) association = UserPolicy(**projection.model_dump()) session.add(association) session.flush() mapping_id = session.execute( insert(LegacyUserPolicyMapping) .values( - country_id=snapshot.country_id, - legacy_user_policy_id=snapshot.legacy_user_policy_id, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, user_policy_id=association.id, last_applied_source_revision=source_revision, fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, @@ -323,7 +247,7 @@ def persist_legacy_user_policy( if mapping_id is not None: return LegacyUserPolicyPersistenceResult( association_id=association.id, - policy_id=policy_result.policy_id, + policy_id=projection.policy_id, association_created=True, association_updated=False, mapping_created=True, @@ -331,18 +255,24 @@ def persist_legacy_user_policy( session.delete(association) session.flush() - concurrent = _mapping(session, snapshot, lock=False) + concurrent = _mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + lock=False, + ) if concurrent is None: raise LegacyUserPolicyIntegrityError( "legacy user-policy mapping conflict did not resolve to a stored row" ) - return _apply_existing_mapping( + return apply_existing_legacy_user_policy_mapping( session, mapping=concurrent, - snapshot=snapshot, + country_id=country_id, + reform_label=reform_label, fingerprint=fingerprint, - user_id=user_id, - policy_id=policy_result.policy_id, + user_id=projection.user_id, + policy_id=projection.policy_id, changed_fields=changed_fields, source_revision=source_revision, ) diff --git a/policyengine_api/data/v2/user_policies/query.py b/policyengine_api/data/v2/user_policies/read_repository.py similarity index 97% rename from policyengine_api/data/v2/user_policies/query.py rename to policyengine_api/data/v2/user_policies/read_repository.py index c051be0d4..4deccb394 100644 --- a/policyengine_api/data/v2/user_policies/query.py +++ b/policyengine_api/data/v2/user_policies/read_repository.py @@ -1,4 +1,4 @@ -"""Country-scoped v2 user-policy association reads.""" +"""Country-scoped database reads for v2 user-policy associations.""" from __future__ import annotations diff --git a/policyengine_api/data/v2/user_policies/persistence.py b/policyengine_api/data/v2/user_policies/write_repository.py similarity index 92% rename from policyengine_api/data/v2/user_policies/persistence.py rename to policyengine_api/data/v2/user_policies/write_repository.py index adf22a99b..4afa638fa 100644 --- a/policyengine_api/data/v2/user_policies/persistence.py +++ b/policyengine_api/data/v2/user_policies/write_repository.py @@ -1,4 +1,4 @@ -"""Transactional persistence for mutable user-policy associations.""" +"""Transactional database writes for mutable user-policy associations.""" from __future__ import annotations @@ -8,12 +8,12 @@ from policyengine_api.data.v2.models import Policy, User, UserPolicy from policyengine_api.data.v2.models.base import utc_now -from policyengine_api.data.v2.user_policies.query import ( +from policyengine_api.data.v2.user_policies.read_repository import ( UserPolicyRead, association_read, get_user_policy_row, ) -from policyengine_api.data.v2.user_policies.schemas import ( +from policyengine_api.services.v2.user_policies.commands import ( UserPolicyCreateCommand, UserPolicyPatchCommand, ) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 2d06c3ef8..5ca4230ce 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -13,7 +13,7 @@ from policyengine_api.json_types import JSONObject if TYPE_CHECKING: - from policyengine_api.data.v2.catalog.schemas import ( + from policyengine_api.data.v2.metadata.read_models import ( MetadataCanonicalParameterValue, MetadataDataset, MetadataDetailResult, @@ -27,17 +27,19 @@ MetadataRegion, MetadataVariable, ) - from policyengine_api.data.v2.policies.query import PolicyPage, PolicyRead - from policyengine_api.data.v2.policies.schemas import NativePolicyCreateCommand - from policyengine_api.data.v2.user_policies.query import ( + from policyengine_api.data.v2.policies.read_repository import PolicyPage, PolicyRead + from policyengine_api.data.v2.user_policies.read_repository import ( UserPolicyPage, UserPolicyRead, ) - from policyengine_api.data.v2.user_policies.schemas import ( + from policyengine_api.services.v2.policies.commands import ( + NativePolicyCreateCommand, + ) + from policyengine_api.services.v2.policies.service import NativePolicyCreation + from policyengine_api.services.v2.user_policies.commands import ( UserPolicyCreateCommand, UserPolicyPatchCommand, ) - from policyengine_api.services.v2.policy_service import NativePolicyCreation class MetadataReader(Protocol): @@ -297,7 +299,7 @@ def _running_policyengine_version() -> str: def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.metadata_service import V2MetadataService + from policyengine_api.services.v2.metadata.service import V2MetadataService return V2MetadataService( get_v2_session_factory()(), @@ -307,7 +309,7 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: def _default_v2_policy_service_factory() -> V2PolicyResourceService: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.policy_service import V2PolicyService + from policyengine_api.services.v2.policies.service import V2PolicyService return V2PolicyService( get_v2_session_factory(), @@ -317,7 +319,7 @@ def _default_v2_policy_service_factory() -> V2PolicyResourceService: def _default_v2_user_policy_service_factory() -> V2UserPolicyResourceService: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.user_policy_service import V2UserPolicyService + from policyengine_api.services.v2.user_policies.service import V2UserPolicyService return V2UserPolicyService(get_v2_session_factory()) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 0cedad8b1..cf6fa1a3b 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -8,7 +8,7 @@ from pydantic import BaseModel from starlette.responses import JSONResponse -from policyengine_api.services.v2.metadata_service import ( +from policyengine_api.services.v2.metadata.service import ( InvalidMetadataPageError, InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, @@ -16,7 +16,9 @@ MetadataResourceNotFoundError, UnsupportedPreviewCountryError, ) -from policyengine_api.data.v2.catalog.schemas import MetadataErrorResponse +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataErrorResponse, +) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import ( NativeRouteDependencies, diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index 82d8539cd..e9a7705b6 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -8,13 +8,13 @@ from fastapi import APIRouter, Query from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.data.v2.metadata.read_models import MetadataRegionType +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataDatasetDetailResponse, MetadataDatasetPageResponse, MetadataEconomyOptionsResponse, MetadataRegionDetailResponse, MetadataRegionPageResponse, - MetadataRegionType, ) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies from policyengine_api.fastapi_routes.v2.metadata.common import ( diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 258c3edfa..01693f512 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Query from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataModelDetailResponse, MetadataModelPageResponse, MetadataModelSelectionResponse, diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 842a1e5fb..714c3e28d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Query from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataParameterChildPageResponse, MetadataParameterDetailResponse, MetadataParameterPageResponse, diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py new file mode 100644 index 000000000..9d0d00a2e --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -0,0 +1,141 @@ +"""Strict HTTP response models for API v2 metadata resources.""" + +from __future__ import annotations + +from typing import Annotated, Generic, Literal, TypeVar + +from pydantic import StringConstraints + +from policyengine_api.data.v2.metadata.read_models import ( + MetadataCanonicalParameterValue, + MetadataDataset, + MetadataDetailResult, + MetadataEconomyOptionsResult, + MetadataModel, + MetadataModelSelectionResult, + MetadataModelVersionDetail, + MetadataPageResult, + MetadataParameterChild, + MetadataParameterSummary, + MetadataRegion, + MetadataVariable, + StrictResponseModel, +) + + +ResourceT = TypeVar("ResourceT") + + +class MetadataResourceSuccessResponse(StrictResponseModel, Generic[ResourceT]): + status: Literal["ok"] = "ok" + message: None = None + result: ResourceT + + +class MetadataModelPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataModel]] +): + pass + + +class MetadataModelDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataModel]] +): + pass + + +class MetadataModelSelectionResponse( + MetadataResourceSuccessResponse[MetadataModelSelectionResult] +): + pass + + +class MetadataModelVersionPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataModelVersionDetail]] +): + pass + + +class MetadataModelVersionDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataModelVersionDetail]] +): + pass + + +class MetadataVariablePageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataVariable]] +): + pass + + +class MetadataVariableDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataVariable]] +): + pass + + +class MetadataParameterPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataParameterSummary]] +): + pass + + +class MetadataParameterDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataParameterSummary]] +): + pass + + +class MetadataParameterChildPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataParameterChild]] +): + pass + + +class MetadataParameterValuePageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataCanonicalParameterValue]] +): + pass + + +class MetadataParameterValueDetailResponse( + MetadataResourceSuccessResponse[ + MetadataDetailResult[MetadataCanonicalParameterValue] + ] +): + pass + + +class MetadataDatasetPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataDataset]] +): + pass + + +class MetadataDatasetDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataDataset]] +): + pass + + +class MetadataRegionPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataRegion]] +): + pass + + +class MetadataRegionDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataRegion]] +): + pass + + +class MetadataEconomyOptionsResponse( + MetadataResourceSuccessResponse[MetadataEconomyOptionsResult] +): + pass + + +class MetadataErrorResponse(StrictResponseModel): + status: Literal["error"] = "error" + message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] diff --git a/policyengine_api/fastapi_routes/v2/policies/__init__.py b/policyengine_api/fastapi_routes/v2/policies/__init__.py new file mode 100644 index 000000000..73b4861b0 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/policies/__init__.py @@ -0,0 +1 @@ +"""FastAPI adapters for immutable v2 policies.""" diff --git a/policyengine_api/fastapi_routes/v2/policies/request_models.py b/policyengine_api/fastapi_routes/v2/policies/request_models.py new file mode 100644 index 000000000..b44e9de5d --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/policies/request_models.py @@ -0,0 +1,10 @@ +"""Strict HTTP request models for the native v2 policy API.""" + +from policyengine_api.services.v2.policies.commands import PolicyCreateCommand + + +MAXIMUM_POLICY_REQUEST_BYTES = 1_048_576 + + +class PolicyCreateRequest(PolicyCreateCommand): + """Native body containing immutable policy content only.""" diff --git a/policyengine_api/data/v2/policies/api_schemas.py b/policyengine_api/fastapi_routes/v2/policies/response_models.py similarity index 89% rename from policyengine_api/data/v2/policies/api_schemas.py rename to policyengine_api/fastapi_routes/v2/policies/response_models.py index 27b70e7fd..6bc31a6b0 100644 --- a/policyengine_api/data/v2/policies/api_schemas.py +++ b/policyengine_api/fastapi_routes/v2/policies/response_models.py @@ -1,4 +1,4 @@ -"""Strict HTTP schemas for the native v2 policy API.""" +"""Strict HTTP response models for the native v2 policy API.""" from __future__ import annotations @@ -8,11 +8,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue, StringConstraints -from policyengine_api.data.v2.policies.query import PolicyPage, PolicyRead -from policyengine_api.data.v2.policies.schemas import PolicyCreateCommand - - -MAXIMUM_POLICY_REQUEST_BYTES = 1_048_576 +from policyengine_api.data.v2.policies.read_repository import PolicyPage, PolicyRead class StrictPolicyAPIModel(BaseModel): @@ -25,10 +21,6 @@ class StrictPolicyAPIModel(BaseModel): ) -class PolicyCreateRequest(PolicyCreateCommand): - """Native body containing immutable policy content only.""" - - class PolicyParameterValueItem(StrictPolicyAPIModel): id: UUID parameter_id: UUID diff --git a/policyengine_api/fastapi_routes/v2/policy_routes.py b/policyengine_api/fastapi_routes/v2/policies/routes.py similarity index 93% rename from policyengine_api/fastapi_routes/v2/policy_routes.py rename to policyengine_api/fastapi_routes/v2/policies/routes.py index 6cc1f4761..c8d8cfe34 100644 --- a/policyengine_api/fastapi_routes/v2/policy_routes.py +++ b/policyengine_api/fastapi_routes/v2/policies/routes.py @@ -14,10 +14,12 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.api_schemas import ( +from policyengine_api.fastapi_routes.v2.policies.request_models import ( MAXIMUM_POLICY_REQUEST_BYTES, - POLICY_ERROR_RESPONSES, PolicyCreateRequest, +) +from policyengine_api.fastapi_routes.v2.policies.response_models import ( + POLICY_ERROR_RESPONSES, PolicyDetailResponse, PolicyDetailResult, PolicyErrorResponse, @@ -25,13 +27,14 @@ PolicyPageResponse, PolicyPageResult, ) -from policyengine_api.data.v2.policies.catalog import PolicyCatalogValidationError -from policyengine_api.data.v2.policies.persistence import ( +from policyengine_api.data.v2.policies.catalog_repository import ( + PolicyCatalogValidationError, +) +from policyengine_api.data.v2.policies.read_repository import PolicyNotFoundError +from policyengine_api.data.v2.policies.write_repository import ( PolicyContentHashCollisionError, PolicyPersistenceIntegrityError, ) -from policyengine_api.data.v2.policies.query import PolicyNotFoundError -from policyengine_api.data.v2.policies.schemas import NativePolicyCreateCommand from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import ( NativeRouteDependencies, @@ -43,6 +46,7 @@ PolicyCreateQuery, PolicyDetailQuery, ) +from policyengine_api.services.v2.policies.commands import NativePolicyCreateCommand class PolicyRequestTooLargeError(ValueError): diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 235287898..aa54f150c 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -5,7 +5,9 @@ from fastapi import APIRouter, Request from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.schemas import MetadataErrorResponse +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataErrorResponse, +) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies from policyengine_api.fastapi_routes.v2.metadata.geography_routes import ( build_v2_metadata_geography_router, @@ -16,8 +18,8 @@ from policyengine_api.fastapi_routes.v2.metadata.parameter_routes import ( build_v2_metadata_parameter_router, ) -from policyengine_api.fastapi_routes.v2.policy_routes import build_v2_policy_router -from policyengine_api.fastapi_routes.v2.user_policy_routes import ( +from policyengine_api.fastapi_routes.v2.policies.routes import build_v2_policy_router +from policyengine_api.fastapi_routes.v2.user_policies.routes import ( build_v2_user_policy_router, ) diff --git a/policyengine_api/fastapi_routes/v2/user_policies/__init__.py b/policyengine_api/fastapi_routes/v2/user_policies/__init__.py new file mode 100644 index 000000000..180dc6780 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/user_policies/__init__.py @@ -0,0 +1 @@ +"""FastAPI adapters for mutable v2 user-policy associations.""" diff --git a/policyengine_api/fastapi_routes/v2/user_policies/request_models.py b/policyengine_api/fastapi_routes/v2/user_policies/request_models.py new file mode 100644 index 000000000..e568d3d4b --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/user_policies/request_models.py @@ -0,0 +1,14 @@ +"""Strict HTTP request models for native v2 user-policy associations.""" + +from policyengine_api.services.v2.user_policies.commands import ( + UserPolicyCreateCommand, + UserPolicyPatchCommand, +) + + +class UserPolicyCreateRequest(UserPolicyCreateCommand): + """Association identity, immutable link fields, and presentation fields.""" + + +class UserPolicyPatchRequest(UserPolicyPatchCommand): + """Explicitly supplied mutable presentation fields.""" diff --git a/policyengine_api/data/v2/user_policies/api_schemas.py b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py similarity index 85% rename from policyengine_api/data/v2/user_policies/api_schemas.py rename to policyengine_api/fastapi_routes/v2/user_policies/response_models.py index d6c3b57b5..eb18fa8f3 100644 --- a/policyengine_api/data/v2/user_policies/api_schemas.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py @@ -1,4 +1,4 @@ -"""Strict HTTP schemas for native v2 user-policy associations.""" +"""Strict HTTP response models for native v2 user-policy associations.""" from __future__ import annotations @@ -8,14 +8,10 @@ from pydantic import BaseModel, ConfigDict, StringConstraints -from policyengine_api.data.v2.user_policies.query import ( +from policyengine_api.data.v2.user_policies.read_repository import ( UserPolicyPage, UserPolicyRead, ) -from policyengine_api.data.v2.user_policies.schemas import ( - UserPolicyCreateCommand, - UserPolicyPatchCommand, -) from policyengine_api.query_parameters import CountryId, ResourceId, UserId @@ -25,14 +21,6 @@ class StrictUserPolicyAPIModel(BaseModel): model_config = ConfigDict(extra="forbid", from_attributes=True) -class UserPolicyCreateRequest(UserPolicyCreateCommand): - """Association identity, immutable link fields, and presentation fields.""" - - -class UserPolicyPatchRequest(UserPolicyPatchCommand): - """Explicitly supplied mutable presentation fields.""" - - class UserPolicyItem(StrictUserPolicyAPIModel): id: UUID country_id: CountryId diff --git a/policyengine_api/fastapi_routes/v2/user_policy_routes.py b/policyengine_api/fastapi_routes/v2/user_policies/routes.py similarity index 95% rename from policyengine_api/fastapi_routes/v2/user_policy_routes.py rename to policyengine_api/fastapi_routes/v2/user_policies/routes.py index ad4cbf43d..5725023ea 100644 --- a/policyengine_api/fastapi_routes/v2/user_policy_routes.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/routes.py @@ -11,23 +11,27 @@ from starlette.responses import JSONResponse, Response from policyengine_api.data.v2.settings import V2ConfigurationError -from policyengine_api.data.v2.user_policies.api_schemas import ( - USER_POLICY_ERROR_RESPONSES, +from policyengine_api.fastapi_routes.v2.user_policies.request_models import ( UserPolicyCreateRequest, + UserPolicyPatchRequest, +) +from policyengine_api.fastapi_routes.v2.user_policies.response_models import ( + USER_POLICY_ERROR_RESPONSES, UserPolicyDetailResponse, UserPolicyDetailResult, UserPolicyErrorResponse, UserPolicyItem, UserPolicyPageResponse, UserPolicyPageResult, - UserPolicyPatchRequest, ) -from policyengine_api.data.v2.user_policies.persistence import ( +from policyengine_api.data.v2.user_policies.read_repository import ( + UserPolicyNotFoundError, +) +from policyengine_api.data.v2.user_policies.write_repository import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, ) -from policyengine_api.data.v2.user_policies.query import UserPolicyNotFoundError from policyengine_api.fastapi_routes.dependencies import ( NativeRouteDependencies, V2UserPolicyResourceService, diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py index d6313537c..48432b523 100644 --- a/policyengine_api/services/policy_mirroring.py +++ b/policyengine_api/services/policy_mirroring.py @@ -12,17 +12,23 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.catalog import PolicyCatalogValidationError -from policyengine_api.data.v2.policies.legacy import ( +from policyengine_api.data.v2.policies.catalog_repository import ( + PolicyCatalogValidationError, +) +from policyengine_api.data.v2.policies.legacy_mapping_repository import ( LegacyPolicyMappingIntegrityError, - LegacyPolicyPersistenceResult, - LegacyPolicySnapshot, - LegacyPolicyTranslationError, ) -from policyengine_api.data.v2.policies.persistence import ( +from policyengine_api.data.v2.policies.write_repository import ( PolicyContentHashCollisionError, PolicyPersistenceIntegrityError, ) +from policyengine_api.services.v2.policies.legacy_service import ( + LegacyPolicyPersistenceResult, +) +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, + LegacyPolicyTranslationError, +) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.gcp_logging import logger @@ -42,7 +48,7 @@ class PolicyMirrorUnavailableError(RuntimeError): def _default_mirror_factory() -> LegacyPolicyMirror: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.policy_service import V2PolicyService + from policyengine_api.services.v2.policies.service import V2PolicyService return V2PolicyService(get_v2_session_factory()) diff --git a/policyengine_api/services/policy_service.py b/policyengine_api/services/policy_service.py index ad6b36f14..e94e97ee9 100644 --- a/policyengine_api/services/policy_service.py +++ b/policyengine_api/services/policy_service.py @@ -11,7 +11,9 @@ from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import Policy -from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, +) from policyengine_api.utils import hash_object diff --git a/policyengine_api/services/user_policy_mirroring.py b/policyengine_api/services/user_policy_mirroring.py index 5f63a796a..e28e39f95 100644 --- a/policyengine_api/services/user_policy_mirroring.py +++ b/policyengine_api/services/user_policy_mirroring.py @@ -12,20 +12,26 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.catalog import PolicyCatalogValidationError -from policyengine_api.data.v2.policies.legacy import ( +from policyengine_api.data.v2.policies.catalog_repository import ( + PolicyCatalogValidationError, +) +from policyengine_api.data.v2.policies.legacy_mapping_repository import ( LegacyPolicyMappingIntegrityError, - LegacyPolicySnapshot, - LegacyPolicyTranslationError, ) -from policyengine_api.data.v2.policies.persistence import ( +from policyengine_api.data.v2.policies.write_repository import ( PolicyContentHashCollisionError, PolicyPersistenceIntegrityError, ) -from policyengine_api.data.v2.settings import V2ConfigurationError -from policyengine_api.data.v2.user_policies.legacy import ( +from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( LegacyUserPolicyIntegrityError, LegacyUserPolicyPersistenceResult, +) +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, + LegacyPolicyTranslationError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, ) from policyengine_api.gcp_logging import logger @@ -53,7 +59,9 @@ class UserPolicyMirrorUnavailableError(RuntimeError): def _default_mirror_factory() -> LegacyUserPolicyMirror: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.user_policy_service import V2UserPolicyService + from policyengine_api.services.v2.user_policies.service import ( + V2UserPolicyService, + ) return V2UserPolicyService(get_v2_session_factory()) diff --git a/policyengine_api/services/user_policy_service.py b/policyengine_api/services/user_policy_service.py index a67b8d54a..d175381fe 100644 --- a/policyengine_api/services/user_policy_service.py +++ b/policyengine_api/services/user_policy_service.py @@ -17,8 +17,10 @@ from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent -from policyengine_api.data.v2.user_policies.legacy import ( +from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( LegacyUserPolicyPersistenceResult, +) +from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, fingerprint_legacy_user_policy, ) diff --git a/policyengine_api/services/v2/metadata/__init__.py b/policyengine_api/services/v2/metadata/__init__.py new file mode 100644 index 000000000..fcf76cce7 --- /dev/null +++ b/policyengine_api/services/v2/metadata/__init__.py @@ -0,0 +1 @@ +"""Application services for API v2 metadata reads.""" diff --git a/policyengine_api/services/v2/metadata_service.py b/policyengine_api/services/v2/metadata/service.py similarity index 51% rename from policyengine_api/services/v2/metadata_service.py rename to policyengine_api/services/v2/metadata/service.py index 93b894b7a..36f29ae94 100644 --- a/policyengine_api/services/v2/metadata_service.py +++ b/policyengine_api/services/v2/metadata/service.py @@ -9,16 +9,26 @@ UnsupportedPreviewCountryError, validate_policyengine_version, ) -from policyengine_api.data.v2.catalog.dataset_query import DatasetQueryMethods -from policyengine_api.data.v2.catalog.model_query import ModelQueryMethods -from policyengine_api.data.v2.catalog.parameter_query import ParameterQueryMethods -from policyengine_api.data.v2.catalog.query_support import ( +from policyengine_api.data.v2.metadata.dataset_read_repository import ( + DatasetReadRepository, +) +from policyengine_api.data.v2.metadata.model_read_repository import ( + ModelReadRepository, +) +from policyengine_api.data.v2.metadata.parameter_read_repository import ( + ParameterReadRepository, +) +from policyengine_api.data.v2.metadata.read_repository import ( InvalidMetadataPageError, MetadataResourceNotFoundError, validate_metadata_page, ) -from policyengine_api.data.v2.catalog.region_query import RegionQueryMethods -from policyengine_api.data.v2.catalog.variable_query import VariableQueryMethods +from policyengine_api.data.v2.metadata.region_read_repository import ( + RegionReadRepository, +) +from policyengine_api.data.v2.metadata.variable_read_repository import ( + VariableReadRepository, +) __all__ = [ @@ -35,10 +45,10 @@ class V2MetadataService( - ModelQueryMethods, - VariableQueryMethods, - ParameterQueryMethods, - DatasetQueryMethods, - RegionQueryMethods, + ModelReadRepository, + VariableReadRepository, + ParameterReadRepository, + DatasetReadRepository, + RegionReadRepository, ): - """Combine the resource-specific query methods into the route-facing API.""" + """Expose resource-specific metadata repositories to the route layer.""" diff --git a/policyengine_api/services/v2/policies/__init__.py b/policyengine_api/services/v2/policies/__init__.py new file mode 100644 index 000000000..391a86685 --- /dev/null +++ b/policyengine_api/services/v2/policies/__init__.py @@ -0,0 +1 @@ +"""Application services for immutable v2 policies.""" diff --git a/policyengine_api/data/v2/policies/schemas.py b/policyengine_api/services/v2/policies/commands.py similarity index 100% rename from policyengine_api/data/v2/policies/schemas.py rename to policyengine_api/services/v2/policies/commands.py diff --git a/policyengine_api/services/v2/policies/legacy_service.py b/policyengine_api/services/v2/policies/legacy_service.py new file mode 100644 index 000000000..ad66e079d --- /dev/null +++ b/policyengine_api/services/v2/policies/legacy_service.py @@ -0,0 +1,109 @@ +"""Transactional operations for mirroring committed v1 policies into v2.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from uuid import UUID + +from sqlmodel import Session + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION +from policyengine_api.data.v2.policies.legacy_mapping_repository import ( + LegacyPolicyMappingIntegrityError, + find_legacy_policy_mapping, + insert_legacy_policy_mapping, + verify_legacy_policy_mapping, +) +from policyengine_api.data.v2.policies.write_repository import ( + persist_resolved_policy, +) +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, + translate_legacy_policy, +) + + +@dataclass(frozen=True) +class LegacyPolicyPersistenceResult: + """Destination identity and insertion outcomes for one mirror attempt.""" + + policy_id: UUID + policy_created: bool + mapping_created: bool + + +def persist_legacy_policy( + session: Session, + snapshot: LegacyPolicySnapshot, + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + country_package_versions: Mapping[str, str] = COUNTRY_PACKAGE_VERSIONS, +) -> LegacyPolicyPersistenceResult: + """Translate, deduplicate, and map one v1 policy in the caller transaction.""" + + existing = find_legacy_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + lock=True, + ) + if existing is not None: + verify_legacy_policy_mapping( + existing, + source_policy_hash=snapshot.source_policy_hash, + ) + + command = translate_legacy_policy( + session, + snapshot, + running_policyengine_version=running_policyengine_version, + country_package_versions=country_package_versions, + ) + policy_result = persist_resolved_policy(session, command) + if existing is not None: + verify_legacy_policy_mapping( + existing, + source_policy_hash=snapshot.source_policy_hash, + expected_policy_id=policy_result.policy_id, + ) + return LegacyPolicyPersistenceResult( + policy_id=existing.policy_id, + policy_created=False, + mapping_created=False, + ) + + mapping_id = insert_legacy_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + source_policy_hash=snapshot.source_policy_hash, + policy_id=policy_result.policy_id, + ) + if mapping_id is not None: + return LegacyPolicyPersistenceResult( + policy_id=policy_result.policy_id, + policy_created=policy_result.created, + mapping_created=True, + ) + + concurrent = find_legacy_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + lock=False, + ) + if concurrent is None: + raise LegacyPolicyMappingIntegrityError( + "legacy policy mapping conflict did not resolve to a stored row" + ) + verify_legacy_policy_mapping( + concurrent, + source_policy_hash=snapshot.source_policy_hash, + expected_policy_id=policy_result.policy_id, + ) + return LegacyPolicyPersistenceResult( + policy_id=concurrent.policy_id, + policy_created=False, + mapping_created=False, + ) diff --git a/policyengine_api/data/v2/policies/legacy.py b/policyengine_api/services/v2/policies/legacy_translation.py similarity index 59% rename from policyengine_api/data/v2/policies/legacy.py rename to policyengine_api/services/v2/policies/legacy_translation.py index e15dc9a58..1f6830aa9 100644 --- a/policyengine_api/data/v2/policies/legacy.py +++ b/policyengine_api/services/v2/policies/legacy_translation.py @@ -1,9 +1,8 @@ -"""Translation of committed v1 policy snapshots into v2 policy commands.""" +"""Translate committed v1 policy snapshots into v2 policy commands.""" from __future__ import annotations from collections.abc import Mapping -from dataclasses import dataclass from datetime import date, datetime, time, timezone from typing import Annotated from uuid import UUID @@ -12,41 +11,28 @@ period as parse_policyengine_period, ) from pydantic import Field, field_validator -from sqlalchemy.dialects.postgresql import insert from sqlmodel import Session, col, select from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION from policyengine_api.data.v2.catalog.catalog_selection import select_catalog -from policyengine_api.data.v2.models import LegacyPolicyMapping, Parameter -from policyengine_api.data.v2.policies.catalog import resolve_policy_catalog -from policyengine_api.data.v2.policies.persistence import persist_resolved_policy -from policyengine_api.data.v2.policies.schemas import ( +from policyengine_api.data.v2.models import Parameter +from policyengine_api.data.v2.policies.catalog_repository import ( + resolve_policy_catalog, +) +from policyengine_api.query_parameters import CountryId +from policyengine_api.services.v2.policies.commands import ( PolicyCreateCommand, PolicyParameterValueCommand, ResolvedPolicyCreateCommand, StrictJsonValue, StrictPolicyCommand, ) -from policyengine_api.query_parameters import CountryId class LegacyPolicyTranslationError(ValueError): """Raised when committed v1 content cannot be interpreted exactly.""" -class LegacyPolicyMappingIntegrityError(RuntimeError): - """Raised when one immutable v1 identity changes or maps inconsistently.""" - - -@dataclass(frozen=True) -class LegacyPolicyPersistenceResult: - """Destination identity and insertion outcomes for one mirror attempt.""" - - policy_id: UUID - policy_created: bool - mapping_created: bool - - class LegacyPolicySnapshot(StrictPolicyCommand): """Detached committed fields required by the v2 policy mirror.""" @@ -70,7 +56,7 @@ def _utc_midnight(value: str) -> datetime: parsed = date.fromisoformat(value) except ValueError as error: raise LegacyPolicyTranslationError( - f"legacy period date {value!r} is invalid" + f"legacy period {value!r} is invalid" ) from error return datetime.combine(parsed, time.min, tzinfo=timezone.utc) @@ -189,101 +175,3 @@ def translate_legacy_policy( command, running_policyengine_version=running_policyengine_version, ) - - -def _legacy_mapping( - session: Session, - snapshot: LegacyPolicySnapshot, - *, - lock: bool, -) -> LegacyPolicyMapping | None: - statement = select(LegacyPolicyMapping).where( - LegacyPolicyMapping.country_id == snapshot.country_id, - LegacyPolicyMapping.legacy_policy_id == snapshot.legacy_policy_id, - ) - if lock: - statement = statement.with_for_update() - return session.exec(statement).one_or_none() - - -def _verify_legacy_mapping( - mapping: LegacyPolicyMapping, - snapshot: LegacyPolicySnapshot, - *, - expected_policy_id: UUID | None = None, -) -> None: - if mapping.source_policy_hash != snapshot.source_policy_hash: - raise LegacyPolicyMappingIntegrityError( - "legacy policy identity was presented with a different source hash" - ) - if expected_policy_id is not None and mapping.policy_id != expected_policy_id: - raise LegacyPolicyMappingIntegrityError( - "legacy policy mapping does not match translated immutable content" - ) - - -def persist_legacy_policy( - session: Session, - snapshot: LegacyPolicySnapshot, - *, - running_policyengine_version: str = POLICYENGINE_VERSION, - country_package_versions: Mapping[str, str] = COUNTRY_PACKAGE_VERSIONS, -) -> LegacyPolicyPersistenceResult: - """Translate, deduplicate, and map one v1 policy in the caller transaction.""" - - existing = _legacy_mapping(session, snapshot, lock=True) - if existing is not None: - _verify_legacy_mapping(existing, snapshot) - - command = translate_legacy_policy( - session, - snapshot, - running_policyengine_version=running_policyengine_version, - country_package_versions=country_package_versions, - ) - policy_result = persist_resolved_policy(session, command) - if existing is not None: - _verify_legacy_mapping( - existing, - snapshot, - expected_policy_id=policy_result.policy_id, - ) - return LegacyPolicyPersistenceResult( - policy_id=existing.policy_id, - policy_created=False, - mapping_created=False, - ) - - mapping_id = session.execute( - insert(LegacyPolicyMapping) - .values( - country_id=snapshot.country_id, - legacy_policy_id=snapshot.legacy_policy_id, - policy_id=policy_result.policy_id, - source_policy_hash=snapshot.source_policy_hash, - ) - .on_conflict_do_nothing(constraint="uq_legacy_policy_mappings_country_legacy") - .returning(col(LegacyPolicyMapping.id)) - ).scalar_one_or_none() - if mapping_id is not None: - return LegacyPolicyPersistenceResult( - policy_id=policy_result.policy_id, - policy_created=policy_result.created, - mapping_created=True, - ) - - concurrent = _legacy_mapping(session, snapshot, lock=False) - if concurrent is None: - raise LegacyPolicyMappingIntegrityError( - "legacy policy mapping conflict did not resolve to a stored row" - ) - _verify_legacy_mapping( - concurrent, - snapshot, - expected_policy_id=policy_result.policy_id, - ) - return LegacyPolicyPersistenceResult( - policy_id=concurrent.policy_id, - policy_created=False, - mapping_created=False, - ) diff --git a/policyengine_api/services/v2/policy_service.py b/policyengine_api/services/v2/policies/service.py similarity index 87% rename from policyengine_api/services/v2/policy_service.py rename to policyengine_api/services/v2/policies/service.py index 9eabf2e6f..7a6604ee9 100644 --- a/policyengine_api/services/v2/policy_service.py +++ b/policyengine_api/services/v2/policies/service.py @@ -9,23 +9,25 @@ from sqlmodel import Session from policyengine_api.constants import POLICYENGINE_VERSION -from policyengine_api.data.v2.policies.catalog import resolve_policy_catalog -from policyengine_api.data.v2.policies.legacy import ( - LegacyPolicyPersistenceResult, - LegacyPolicySnapshot, - persist_legacy_policy, -) -from policyengine_api.data.v2.policies.persistence import persist_resolved_policy -from policyengine_api.data.v2.policies.query import ( +from policyengine_api.data.v2.policies.catalog_repository import resolve_policy_catalog +from policyengine_api.data.v2.policies.read_repository import ( PolicyPage, PolicyRead, list_policies, read_policy, ) -from policyengine_api.data.v2.policies.schemas import ( +from policyengine_api.data.v2.policies.write_repository import persist_resolved_policy +from policyengine_api.services.v2.policies.commands import ( NativePolicyCreateCommand, PolicyCreateCommand, ) +from policyengine_api.services.v2.policies.legacy_service import ( + LegacyPolicyPersistenceResult, + persist_legacy_policy, +) +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, +) @dataclass(frozen=True) diff --git a/policyengine_api/services/v2/user_policies/__init__.py b/policyengine_api/services/v2/user_policies/__init__.py new file mode 100644 index 000000000..77569938d --- /dev/null +++ b/policyengine_api/services/v2/user_policies/__init__.py @@ -0,0 +1 @@ +"""Application services for mutable v2 user-policy associations.""" diff --git a/policyengine_api/data/v2/user_policies/schemas.py b/policyengine_api/services/v2/user_policies/commands.py similarity index 100% rename from policyengine_api/data/v2/user_policies/schemas.py rename to policyengine_api/services/v2/user_policies/commands.py diff --git a/policyengine_api/services/v2/user_policies/legacy_service.py b/policyengine_api/services/v2/user_policies/legacy_service.py new file mode 100644 index 000000000..1470e7666 --- /dev/null +++ b/policyengine_api/services/v2/user_policies/legacy_service.py @@ -0,0 +1,67 @@ +"""Transactional operations for mirroring v1 saved policies into v2.""" + +from __future__ import annotations + +from sqlmodel import Session + +from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( + LegacyUserPolicyIntegrityError, + LegacyUserPolicyPersistenceResult, + persist_legacy_user_policy_mapping, + resolve_legacy_user_id, +) +from policyengine_api.services.v2.policies.legacy_service import ( + persist_legacy_policy, +) +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.legacy_translation import ( + LegacyUserPolicySnapshot, + fingerprint_legacy_user_policy, + project_legacy_user_policy, +) + + +def persist_legacy_user_policy( + session: Session, + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str] = frozenset(), +) -> LegacyUserPolicyPersistenceResult: + """Ensure reform, association, and identity mappings in one transaction.""" + + if source_revision <= 0: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy source revision must be positive" + ) + if ( + snapshot.country_id != reform_snapshot.country_id + or snapshot.reform_id != reform_snapshot.legacy_policy_id + ): + raise LegacyUserPolicyIntegrityError( + "saved policy does not reference the supplied reform snapshot" + ) + policy_result = persist_legacy_policy(session, reform_snapshot) + user_id = resolve_legacy_user_id( + session, + legacy_user_id=snapshot.user_id, + primary_country=snapshot.country_id, + ) + projection = project_legacy_user_policy( + snapshot, + user_id=user_id, + policy_id=policy_result.policy_id, + ) + return persist_legacy_user_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + reform_label=snapshot.reform_label, + projection=projection, + fingerprint=fingerprint_legacy_user_policy(snapshot), + source_revision=source_revision, + changed_fields=changed_fields, + ) diff --git a/policyengine_api/services/v2/user_policies/legacy_translation.py b/policyengine_api/services/v2/user_policies/legacy_translation.py new file mode 100644 index 000000000..7bd03b9fc --- /dev/null +++ b/policyengine_api/services/v2/user_policies/legacy_translation.py @@ -0,0 +1,74 @@ +"""Translate committed v1 saved-policy rows into v2 association commands.""" + +from __future__ import annotations + +from hashlib import sha256 +import json +from typing import Annotated +from uuid import UUID + +from pydantic import Field + +from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( + USER_POLICY_FINGERPRINT_VERSION, +) +from policyengine_api.query_parameters import CountryId, LegacyUserId +from policyengine_api.services.v2.policies.commands import StrictPolicyCommand +from policyengine_api.services.v2.user_policies.commands import ( + UserPolicyCreateCommand, +) + + +class LegacyUserPolicySnapshot(StrictPolicyCommand): + """Detached complete committed v1 saved-policy row.""" + + country_id: CountryId + legacy_user_policy_id: Annotated[int, Field(ge=0)] + reform_id: Annotated[int, Field(ge=0)] + reform_label: Annotated[str, Field(max_length=255)] | None = None + baseline_id: Annotated[int, Field(ge=0)] + baseline_label: Annotated[str, Field(max_length=255)] | None = None + user_id: LegacyUserId + year: Annotated[str, Field(max_length=32)] + geography: Annotated[str, Field(max_length=255)] + dataset: Annotated[str, Field(max_length=255)] | None = None + number_of_provisions: Annotated[int, Field(ge=0)] + api_version: Annotated[str, Field(max_length=32)] + added_date: int + updated_date: int + budgetary_impact: Annotated[str, Field(max_length=255)] | None = None + type: Annotated[str, Field(max_length=255)] | None = None + + +def fingerprint_legacy_user_policy(snapshot: LegacyUserPolicySnapshot) -> str: + """Hash every committed source field through deterministic JSON.""" + + document = { + "fingerprint_version": USER_POLICY_FINGERPRINT_VERSION, + **snapshot.model_dump(mode="json"), + } + encoded = json.dumps( + document, + ensure_ascii=True, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def project_legacy_user_policy( + snapshot: LegacyUserPolicySnapshot, + *, + user_id: UUID, + policy_id: UUID, +) -> UserPolicyCreateCommand: + """Map v1 presentation data onto an association, never core policy content.""" + + return UserPolicyCreateCommand( + country_id=snapshot.country_id, + user_id=user_id, + policy_id=policy_id, + name=snapshot.reform_label, + description=None, + ) diff --git a/policyengine_api/services/v2/user_policy_service.py b/policyengine_api/services/v2/user_policies/service.py similarity index 87% rename from policyengine_api/services/v2/user_policy_service.py rename to policyengine_api/services/v2/user_policies/service.py index fc42eae4d..dfe5e85d9 100644 --- a/policyengine_api/services/v2/user_policy_service.py +++ b/policyengine_api/services/v2/user_policies/service.py @@ -7,27 +7,31 @@ from sqlalchemy.orm import sessionmaker from sqlmodel import Session -from policyengine_api.data.v2.user_policies.persistence import ( - create_user_policy, - delete_user_policy, - patch_user_policy, -) -from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot -from policyengine_api.data.v2.user_policies.legacy import ( - LegacyUserPolicyPersistenceResult, - LegacyUserPolicySnapshot, - persist_legacy_user_policy, -) -from policyengine_api.data.v2.user_policies.query import ( +from policyengine_api.data.v2.user_policies.read_repository import ( UserPolicyPage, UserPolicyRead, list_user_policies, read_user_policy, ) -from policyengine_api.data.v2.user_policies.schemas import ( +from policyengine_api.data.v2.user_policies.write_repository import ( + create_user_policy, + delete_user_policy, + patch_user_policy, +) +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.commands import ( UserPolicyCreateCommand, UserPolicyPatchCommand, ) +from policyengine_api.services.v2.user_policies.legacy_service import ( + LegacyUserPolicyPersistenceResult, + persist_legacy_user_policy, +) +from policyengine_api.services.v2.user_policies.legacy_translation import ( + LegacyUserPolicySnapshot, +) class V2UserPolicyService: diff --git a/pyproject.toml b/pyproject.toml index e14d47a2e..2fc4fd990 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,7 @@ files = [ "policyengine_api/fastapi_routes/query_parameters.py", "policyengine_api/fastapi_routes/dependencies.py", "policyengine_api/fastapi_routes/v2", + "policyengine_api/data/v2/metadata", "policyengine_api/data/v2/policies", "policyengine_api/data/v2/user_policies", "policyengine_api/services/v2", diff --git a/tests/contract/test_policy_v2_compatibility.py b/tests/contract/test_policy_v2_compatibility.py index 2ebbd5e81..3a2bd7f39 100644 --- a/tests/contract/test_policy_v2_compatibility.py +++ b/tests/contract/test_policy_v2_compatibility.py @@ -7,8 +7,10 @@ import pytest from policyengine_api.data.v2.models import Policy -from policyengine_api.data.v2.policies.api_schemas import PolicyCreateRequest -from policyengine_api.data.v2.user_policies.legacy import ( +from policyengine_api.fastapi_routes.v2.policies.request_models import ( + PolicyCreateRequest, +) +from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, project_legacy_user_policy, ) diff --git a/tests/contract/test_v1_route_contracts.py b/tests/contract/test_v1_route_contracts.py index 66915b3e3..2a400ad1b 100644 --- a/tests/contract/test_v1_route_contracts.py +++ b/tests/contract/test_v1_route_contracts.py @@ -16,7 +16,9 @@ Simulation, UserPolicy, ) -from policyengine_api.data.v2.user_policies.legacy import LegacyUserPolicySnapshot +from policyengine_api.services.v2.user_policies.legacy_translation import ( + LegacyUserPolicySnapshot, +) from policyengine_api.extensions import cache from policyengine_api.routes.household_routes import household_bp from policyengine_api.routes.policy_routes import policy_bp diff --git a/tests/integration/test_v1_policy_dual_write.py b/tests/integration/test_v1_policy_dual_write.py index 2469f9482..4b8787893 100644 --- a/tests/integration/test_v1_policy_dual_write.py +++ b/tests/integration/test_v1_policy_dual_write.py @@ -24,8 +24,10 @@ TaxBenefitModel, TaxBenefitModelVersion, ) -from policyengine_api.data.v2.policies.legacy import persist_legacy_policy -from policyengine_api.services.v2.policy_service import V2PolicyService +from policyengine_api.services.v2.policies.legacy_service import ( + persist_legacy_policy, +) +from policyengine_api.services.v2.policies.service import V2PolicyService from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from policyengine_api.services.policy_mirroring import ( PolicyMirrorUnavailableError, diff --git a/tests/integration/test_v1_user_policy_dual_write.py b/tests/integration/test_v1_user_policy_dual_write.py index 5a25cc802..4fcd6f5a2 100644 --- a/tests/integration/test_v1_user_policy_dual_write.py +++ b/tests/integration/test_v1_user_policy_dual_write.py @@ -28,7 +28,7 @@ UserPolicy, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL -from policyengine_api.services.v2.user_policy_service import V2UserPolicyService +from policyengine_api.services.v2.user_policies.service import V2UserPolicyService from policyengine_api.services.policy_service import PolicyService from policyengine_api.services.user_policy_mirroring import ( UserPolicyMirrorUnavailableError, diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 765bc6269..d126286d5 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -18,7 +18,7 @@ from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.data.v2.catalog.publication import publish_catalog -from policyengine_api.services.v2.metadata_service import V2MetadataService +from policyengine_api.services.v2.metadata.service import V2MetadataService from policyengine_api.data.v2.models import ( Dataset, TaxBenefitModel, diff --git a/tests/integration/test_v2_policy_persistence.py b/tests/integration/test_v2_policy_persistence.py index 691ab678f..2010f1c08 100644 --- a/tests/integration/test_v2_policy_persistence.py +++ b/tests/integration/test_v2_policy_persistence.py @@ -29,16 +29,22 @@ canonical_policy_document, canonicalize_policy, ) -from policyengine_api.data.v2.policies.persistence import ( +from policyengine_api.data.v2.policies.legacy_mapping_repository import ( + LegacyPolicyMappingIntegrityError, +) +from policyengine_api.data.v2.policies.write_repository import ( PolicyContentHashCollisionError, persist_resolved_policy, ) -from policyengine_api.data.v2.policies.legacy import ( - LegacyPolicyMappingIntegrityError, - LegacyPolicySnapshot, +from policyengine_api.services.v2.policies.commands import ( + ResolvedPolicyCreateCommand, +) +from policyengine_api.services.v2.policies.legacy_service import ( persist_legacy_policy, ) -from policyengine_api.data.v2.policies.schemas import ResolvedPolicyCreateCommand +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, +) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL diff --git a/tests/integration/test_v2_user_policy_mirroring.py b/tests/integration/test_v2_user_policy_mirroring.py index 58b31a8a4..2344884d9 100644 --- a/tests/integration/test_v2_user_policy_mirroring.py +++ b/tests/integration/test_v2_user_policy_mirroring.py @@ -29,13 +29,19 @@ User, UserPolicy, ) -from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot +from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( + resolve_legacy_user_id, +) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL -from policyengine_api.data.v2.user_policies.legacy import ( +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.legacy_service import ( + persist_legacy_user_policy, +) +from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, fingerprint_legacy_user_policy, - persist_legacy_user_policy, - resolve_legacy_user_id, ) diff --git a/tests/unit/routes/test_policy_dual_write_routes.py b/tests/unit/routes/test_policy_dual_write_routes.py index d47667f6f..9994c131a 100644 --- a/tests/unit/routes/test_policy_dual_write_routes.py +++ b/tests/unit/routes/test_policy_dual_write_routes.py @@ -8,7 +8,9 @@ from flask import Flask from policyengine_api.data.v1_models import Policy -from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, +) from policyengine_api.routes.policy_routes import policy_bp from policyengine_api.services.policy_mirroring import PolicyMirrorUnavailableError from policyengine_api.services.policy_service import PolicySetResult diff --git a/tests/unit/routes/test_user_policy_dual_write_routes.py b/tests/unit/routes/test_user_policy_dual_write_routes.py index 881266845..c68e1a9ac 100644 --- a/tests/unit/routes/test_user_policy_dual_write_routes.py +++ b/tests/unit/routes/test_user_policy_dual_write_routes.py @@ -15,8 +15,12 @@ ) from policyengine_api.data.v1_models import UserPolicy -from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot -from policyengine_api.data.v2.user_policies.legacy import LegacyUserPolicySnapshot +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.legacy_translation import ( + LegacyUserPolicySnapshot, +) from policyengine_api.routes.policy_routes import policy_bp from policyengine_api.services.user_policy_mirroring import ( UserPolicyMirrorUnavailableError, diff --git a/tests/unit/services/test_policy_mirroring.py b/tests/unit/services/test_policy_mirroring.py index 908d54f1e..765daf9cf 100644 --- a/tests/unit/services/test_policy_mirroring.py +++ b/tests/unit/services/test_policy_mirroring.py @@ -8,9 +8,13 @@ import pytest from sqlalchemy.exc import OperationalError, TimeoutError -from policyengine_api.data.v2.policies.legacy import ( +from policyengine_api.data.v2.policies.legacy_mapping_repository import ( LegacyPolicyMappingIntegrityError, +) +from policyengine_api.services.v2.policies.legacy_service import ( LegacyPolicyPersistenceResult, +) +from policyengine_api.services.v2.policies.legacy_translation import ( LegacyPolicySnapshot, ) from policyengine_api.services.policy_mirroring import ( diff --git a/tests/unit/services/test_user_policy_mirroring.py b/tests/unit/services/test_user_policy_mirroring.py index d64e4a3ae..cfbf443ab 100644 --- a/tests/unit/services/test_user_policy_mirroring.py +++ b/tests/unit/services/test_user_policy_mirroring.py @@ -10,10 +10,14 @@ from sqlalchemy.exc import OperationalError, TimeoutError from policyengine_api.data.v1_models import UserPolicyMirrorEvent -from policyengine_api.data.v2.policies.legacy import LegacyPolicySnapshot -from policyengine_api.data.v2.user_policies.legacy import ( +from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( LegacyUserPolicyIntegrityError, LegacyUserPolicyPersistenceResult, +) +from policyengine_api.services.v2.policies.legacy_translation import ( + LegacyPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, ) from policyengine_api.services.user_policy_mirroring import ( diff --git a/tests/unit/services/test_user_policy_service.py b/tests/unit/services/test_user_policy_service.py index 83b6380d9..8c85700ce 100644 --- a/tests/unit/services/test_user_policy_service.py +++ b/tests/unit/services/test_user_policy_service.py @@ -14,7 +14,7 @@ ) from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent -from policyengine_api.data.v2.user_policies.legacy import ( +from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( LegacyUserPolicyPersistenceResult, ) from policyengine_api.services.user_policy_service import ( diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 5e4a881e8..99e71b6a5 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -11,7 +11,7 @@ import pytest from policyengine_api.asgi_factory import create_asgi_app -from policyengine_api.services.v2.metadata_service import ( +from policyengine_api.services.v2.metadata.service import ( InvalidMetadataPageError, InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, @@ -19,7 +19,7 @@ MetadataResourceNotFoundError, UnsupportedPreviewCountryError, ) -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.data.v2.metadata.read_models import ( MetadataCanonicalParameterValue, MetadataDataset, MetadataDatasetOption, @@ -435,7 +435,7 @@ def test_default_reader_uses_the_installed_policyengine_version( monkeypatch: pytest.MonkeyPatch, ) -> None: from policyengine_api.data.v2 import database - from policyengine_api.services.v2 import metadata_service + from policyengine_api.services.v2.metadata import service as metadata_service session = object() captured = {} diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 49010b17a..3cd45465e 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -12,7 +12,7 @@ from sqlalchemy.pool import StaticPool from sqlmodel import Session, create_engine, select -from policyengine_api.services.v2.metadata_service import ( +from policyengine_api.services.v2.metadata.service import ( InvalidMetadataPageError, InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, @@ -551,23 +551,21 @@ def test_economy_options_require_a_national_region_and_dataset( _service(catalog_session).get_economy_options("us") -def test_query_modules_import_no_policyengine_or_v1_metadata_source() -> None: - source_directory = ( - Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" / "catalog" - ) +def test_read_repositories_import_no_policyengine_or_v1_metadata_source() -> None: + data_directory = Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" modules = ( - "catalog_selection.py", - "dataset_query.py", - "model_query.py", - "parameter_query.py", - "parameter_tree_query.py", - "query_support.py", - "region_query.py", - "variable_query.py", + data_directory / "catalog" / "catalog_selection.py", + data_directory / "metadata" / "dataset_read_repository.py", + data_directory / "metadata" / "model_read_repository.py", + data_directory / "metadata" / "parameter_read_repository.py", + data_directory / "metadata" / "parameter_tree_read_repository.py", + data_directory / "metadata" / "read_repository.py", + data_directory / "metadata" / "region_read_repository.py", + data_directory / "metadata" / "variable_read_repository.py", ) imported = set() for module in modules: - tree = ast.parse((source_directory / module).read_text(encoding="utf-8")) + tree = ast.parse(module.read_text(encoding="utf-8")) imported.update( alias.name for node in ast.walk(tree) @@ -595,26 +593,26 @@ def test_query_modules_import_no_policyengine_or_v1_metadata_source() -> None: ) -def test_resource_service_methods_are_defined_in_their_query_modules() -> None: +def test_resource_service_methods_are_defined_in_their_read_repositories() -> None: expected_modules = { - "list_models": "model_query", - "get_model": "model_query", - "get_model_by_country": "model_query", - "list_model_versions": "model_query", - "get_model_version": "model_query", - "list_variables": "variable_query", - "get_variable": "variable_query", - "list_parameters": "parameter_query", - "get_parameter": "parameter_query", - "list_parameter_children": "parameter_query", - "list_parameter_values": "parameter_query", - "get_parameter_value": "parameter_query", - "list_datasets": "dataset_query", - "get_dataset": "dataset_query", - "list_regions": "region_query", - "get_region": "region_query", - "get_region_by_code": "region_query", - "get_economy_options": "region_query", + "list_models": "model_read_repository", + "get_model": "model_read_repository", + "get_model_by_country": "model_read_repository", + "list_model_versions": "model_read_repository", + "get_model_version": "model_read_repository", + "list_variables": "variable_read_repository", + "get_variable": "variable_read_repository", + "list_parameters": "parameter_read_repository", + "get_parameter": "parameter_read_repository", + "list_parameter_children": "parameter_read_repository", + "list_parameter_values": "parameter_read_repository", + "get_parameter_value": "parameter_read_repository", + "list_datasets": "dataset_read_repository", + "get_dataset": "dataset_read_repository", + "list_regions": "region_read_repository", + "get_region": "region_read_repository", + "get_region_by_code": "region_read_repository", + "get_economy_options": "region_read_repository", } for method_name, module_name in expected_modules.items(): diff --git a/tests/unit/v2/test_policy_canonicalization.py b/tests/unit/v2/test_policy_canonicalization.py index bf20eee9a..4a361a9b0 100644 --- a/tests/unit/v2/test_policy_canonicalization.py +++ b/tests/unit/v2/test_policy_canonicalization.py @@ -11,7 +11,9 @@ canonical_policy_document, canonicalize_policy, ) -from policyengine_api.data.v2.policies.schemas import ResolvedPolicyCreateCommand +from policyengine_api.services.v2.policies.commands import ( + ResolvedPolicyCreateCommand, +) MODEL_ID = UUID("00000000-0000-0000-0000-000000000010") diff --git a/tests/unit/v2/test_policy_catalog.py b/tests/unit/v2/test_policy_catalog.py index 5b82ae1fd..db72cffbf 100644 --- a/tests/unit/v2/test_policy_catalog.py +++ b/tests/unit/v2/test_policy_catalog.py @@ -16,11 +16,11 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.data.v2.policies.catalog import ( +from policyengine_api.data.v2.policies.catalog_repository import ( PolicyCatalogValidationError, resolve_policy_catalog, ) -from policyengine_api.data.v2.policies.schemas import PolicyCreateCommand +from policyengine_api.services.v2.policies.commands import PolicyCreateCommand def _catalog_session(): diff --git a/tests/unit/v2/test_policy_commands.py b/tests/unit/v2/test_policy_commands.py index 59108bd31..3502df5f9 100644 --- a/tests/unit/v2/test_policy_commands.py +++ b/tests/unit/v2/test_policy_commands.py @@ -9,7 +9,7 @@ from pydantic import ValidationError import pytest -from policyengine_api.data.v2.policies.schemas import ( +from policyengine_api.services.v2.policies.commands import ( MAXIMUM_POLICY_PARAMETER_VALUES, NativePolicyCreateCommand, PolicyCreateCommand, diff --git a/tests/unit/v2/test_policy_legacy_translation.py b/tests/unit/v2/test_policy_legacy_translation.py index f69094a0b..2f29f348e 100644 --- a/tests/unit/v2/test_policy_legacy_translation.py +++ b/tests/unit/v2/test_policy_legacy_translation.py @@ -14,7 +14,7 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.data.v2.policies.legacy import ( +from policyengine_api.services.v2.policies.legacy_translation import ( LegacyPolicySnapshot, LegacyPolicyTranslationError, parse_legacy_period, diff --git a/tests/unit/v2/test_policy_persistence_statements.py b/tests/unit/v2/test_policy_persistence_statements.py index f51716d86..5163e29f3 100644 --- a/tests/unit/v2/test_policy_persistence_statements.py +++ b/tests/unit/v2/test_policy_persistence_statements.py @@ -4,11 +4,11 @@ from sqlalchemy.dialects import postgresql -from policyengine_api.data.v2.policies import persistence +from policyengine_api.data.v2.policies import write_repository def test_policy_insert_uses_the_content_identity_constraint_and_returning() -> None: - source = persistence._insert_policy.__code__.co_consts + source = write_repository._insert_policy.__code__.co_consts statement_text = " ".join(str(value) for value in source) assert "uq_policies_canonicalization_content_hash" in statement_text @@ -16,9 +16,9 @@ def test_policy_insert_uses_the_content_identity_constraint_and_returning() -> N # Compile a representative statement through the same PostgreSQL dialect # construct to prove this module does not use a read-before-write insert. statement = ( - persistence.insert(persistence.Policy) + write_repository.insert(write_repository.Policy) .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") - .returning(persistence.Policy.id) + .returning(write_repository.Policy.id) ) compiled = str(statement.compile(dialect=postgresql.dialect())) assert "ON CONFLICT ON CONSTRAINT" in compiled diff --git a/tests/unit/v2/test_policy_query.py b/tests/unit/v2/test_policy_query.py index 86c4c72b8..2f2616e69 100644 --- a/tests/unit/v2/test_policy_query.py +++ b/tests/unit/v2/test_policy_query.py @@ -16,7 +16,7 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.data.v2.policies.query import ( +from policyengine_api.data.v2.policies.read_repository import ( PolicyNotFoundError, list_policies, read_policy, diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py index aa931739c..2b2a0da4b 100644 --- a/tests/unit/v2/test_policy_routes.py +++ b/tests/unit/v2/test_policy_routes.py @@ -15,24 +15,26 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.catalog import PolicyCatalogValidationError -from policyengine_api.data.v2.policies.persistence import ( - PolicyContentHashCollisionError, - PolicyPersistenceIntegrityError, +from policyengine_api.data.v2.policies.catalog_repository import ( + PolicyCatalogValidationError, ) -from policyengine_api.data.v2.policies.query import ( +from policyengine_api.data.v2.policies.read_repository import ( PolicyNotFoundError, PolicyPage, PolicyParameterValueRead, PolicyRead, ) -from policyengine_api.services.v2.policy_service import NativePolicyCreation +from policyengine_api.data.v2.policies.write_repository import ( + PolicyContentHashCollisionError, + PolicyPersistenceIntegrityError, +) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, ) +from policyengine_api.services.v2.policies.service import NativePolicyCreation POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") diff --git a/tests/unit/v2/test_user_policy_legacy.py b/tests/unit/v2/test_user_policy_legacy.py index 694fcd8eb..69d96293d 100644 --- a/tests/unit/v2/test_user_policy_legacy.py +++ b/tests/unit/v2/test_user_policy_legacy.py @@ -8,11 +8,13 @@ import pytest from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy -from policyengine_api.data.v2.user_policies.legacy import ( - USER_POLICY_FINGERPRINT_VERSION, +from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( LegacyUserPolicyIntegrityError, + apply_existing_legacy_user_policy_mapping, +) +from policyengine_api.services.v2.user_policies.legacy_translation import ( + USER_POLICY_FINGERPRINT_VERSION, LegacyUserPolicySnapshot, - _apply_existing_mapping, fingerprint_legacy_user_policy, ) @@ -113,10 +115,11 @@ def _apply_update( ) session = MagicMock() session.exec.return_value.one_or_none.return_value = association - result = _apply_existing_mapping( + result = apply_existing_legacy_user_policy_mapping( session, mapping=mapping, - snapshot=_snapshot(reform_label="Legacy rename", year="2027"), + country_id="us", + reform_label="Legacy rename", fingerprint=fingerprint, user_id=USER_ID, policy_id=POLICY_ID, diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py index 70d6c9e21..660c3e76b 100644 --- a/tests/unit/v2/test_user_policy_routes.py +++ b/tests/unit/v2/test_user_policy_routes.py @@ -12,16 +12,16 @@ from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.data.v2.settings import V2ConfigurationError -from policyengine_api.data.v2.user_policies.persistence import ( - AssociationCountryConflictError, - AssociationPolicyNotFoundError, - AssociationUserNotFoundError, -) -from policyengine_api.data.v2.user_policies.query import ( +from policyengine_api.data.v2.user_policies.read_repository import ( UserPolicyNotFoundError, UserPolicyPage, UserPolicyRead, ) +from policyengine_api.data.v2.user_policies.write_repository import ( + AssociationCountryConflictError, + AssociationPolicyNotFoundError, + AssociationUserNotFoundError, +) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies from policyengine_api.migration_flags import ( RouteImplementation, diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py index f4ea8f543..fdd37bbb9 100644 --- a/tests/unit/v2/test_user_policy_service.py +++ b/tests/unit/v2/test_user_policy_service.py @@ -21,17 +21,19 @@ UserPolicy, V2_METADATA, ) -from policyengine_api.data.v2.user_policies.persistence import ( +from policyengine_api.data.v2.user_policies.read_repository import ( + UserPolicyNotFoundError, +) +from policyengine_api.data.v2.user_policies.write_repository import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, ) -from policyengine_api.data.v2.user_policies.query import UserPolicyNotFoundError -from policyengine_api.data.v2.user_policies.schemas import ( +from policyengine_api.services.v2.user_policies.commands import ( UserPolicyCreateCommand, UserPolicyPatchCommand, ) -from policyengine_api.services.v2.user_policy_service import V2UserPolicyService +from policyengine_api.services.v2.user_policies.service import V2UserPolicyService USER_ID = UUID("00000000-0000-0000-0000-000000000070") From ae9f76bfa167b61c0e902a500c95d587f3f1e75c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 2 Sep 2026 00:06:16 +0400 Subject: [PATCH 08/18] Use literal names for API v2 SQL modules --- .github/copilot-instructions.md | 3 ++ AGENTS.md | 3 ++ CLAUDE.md | 3 ++ docs/engineering/skills/README.md | 2 + docs/engineering/skills/testing.md | 6 +-- .../{ => skills}/v2-code-organization.md | 47 +++++++++------- policyengine_api/data/v2/metadata/__init__.py | 2 +- ..._read_repository.py => dataset_queries.py} | 6 +-- ...el_read_repository.py => model_queries.py} | 6 +-- ...ead_repository.py => parameter_queries.py} | 8 +-- ...epository.py => parameter_tree_queries.py} | 2 +- .../{read_repository.py => query_support.py} | 6 +-- ...n_read_repository.py => region_queries.py} | 6 +-- ...read_repository.py => variable_queries.py} | 6 +-- ...og_repository.py => catalog_resolution.py} | 2 +- ...pping_repository.py => legacy_mappings.py} | 2 +- .../{write_repository.py => persistence.py} | 2 +- .../{read_repository.py => queries.py} | 2 +- ...pping_repository.py => legacy_mappings.py} | 2 +- .../{write_repository.py => persistence.py} | 4 +- .../{read_repository.py => queries.py} | 2 +- .../fastapi_routes/dependencies.py | 4 +- .../v2/policies/response_models.py | 2 +- .../fastapi_routes/v2/policies/routes.py | 6 +-- .../v2/user_policies/response_models.py | 2 +- .../fastapi_routes/v2/user_policies/routes.py | 4 +- policyengine_api/services/policy_mirroring.py | 6 +-- .../services/user_policy_mirroring.py | 8 +-- .../services/user_policy_service.py | 2 +- .../services/v2/metadata/service.py | 34 ++++++------ .../services/v2/policies/legacy_service.py | 4 +- .../v2/policies/legacy_translation.py | 2 +- .../services/v2/policies/service.py | 6 +-- .../v2/user_policies/legacy_service.py | 2 +- .../v2/user_policies/legacy_translation.py | 2 +- .../services/v2/user_policies/service.py | 4 +- .../integration/test_v2_policy_persistence.py | 4 +- .../test_v2_user_policy_mirroring.py | 2 +- tests/unit/services/test_policy_mirroring.py | 2 +- .../services/test_user_policy_mirroring.py | 2 +- .../unit/services/test_user_policy_service.py | 2 +- tests/unit/v2/test_metadata_service.py | 54 +++++++++---------- tests/unit/v2/test_policy_catalog.py | 2 +- .../v2/test_policy_persistence_statements.py | 8 +-- tests/unit/v2/test_policy_query.py | 2 +- tests/unit/v2/test_policy_routes.py | 6 +-- tests/unit/v2/test_user_policy_legacy.py | 2 +- tests/unit/v2/test_user_policy_routes.py | 4 +- tests/unit/v2/test_user_policy_service.py | 4 +- 49 files changed, 162 insertions(+), 142 deletions(-) rename docs/engineering/{ => skills}/v2-code-organization.md (52%) rename policyengine_api/data/v2/metadata/{dataset_read_repository.py => dataset_queries.py} (94%) rename policyengine_api/data/v2/metadata/{model_read_repository.py => model_queries.py} (96%) rename policyengine_api/data/v2/metadata/{parameter_read_repository.py => parameter_queries.py} (97%) rename policyengine_api/data/v2/metadata/{parameter_tree_read_repository.py => parameter_tree_queries.py} (98%) rename policyengine_api/data/v2/metadata/{read_repository.py => query_support.py} (96%) rename policyengine_api/data/v2/metadata/{region_read_repository.py => region_queries.py} (97%) rename policyengine_api/data/v2/metadata/{variable_read_repository.py => variable_queries.py} (95%) rename policyengine_api/data/v2/policies/{catalog_repository.py => catalog_resolution.py} (97%) rename policyengine_api/data/v2/policies/{legacy_mapping_repository.py => legacy_mappings.py} (96%) rename policyengine_api/data/v2/policies/{write_repository.py => persistence.py} (98%) rename policyengine_api/data/v2/policies/{read_repository.py => queries.py} (98%) rename policyengine_api/data/v2/user_policies/{legacy_mapping_repository.py => legacy_mappings.py} (99%) rename policyengine_api/data/v2/user_policies/{write_repository.py => persistence.py} (95%) rename policyengine_api/data/v2/user_policies/{read_repository.py => queries.py} (97%) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1dcd3de6b..ae1e49a44 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -16,5 +16,8 @@ reviewing test files. For SQLAlchemy model or Alembic migration work, read `docs/engineering/skills/alembic-migrations.md`. +For API v2 route, service, or database-access module additions or moves, read +`docs/engineering/skills/v2-code-organization.md`. + For pull requests, read `docs/engineering/skills/github-prs.md` before opening, replacing, or sharing a PR. diff --git a/AGENTS.md b/AGENTS.md index 9007bf341..a56d5ae68 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,9 @@ When changing SQLAlchemy models, Alembic configuration, database schemas, or migration revisions, read `docs/engineering/skills/alembic-migrations.md`. +When adding or moving API v2 route, service, or database-access modules, read +`docs/engineering/skills/v2-code-organization.md`. + ## GitHub PRs Read `docs/engineering/skills/github-prs.md` before opening, replacing, or diff --git a/CLAUDE.md b/CLAUDE.md index 3e44f755f..74b78e9f8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,6 +30,9 @@ When changing SQLAlchemy models, Alembic configuration, database schemas, or migration revisions, read `docs/engineering/skills/alembic-migrations.md`. +When adding or moving API v2 route, service, or database-access modules, read +`docs/engineering/skills/v2-code-organization.md`. + ## Safety Boundaries Do not claim a route, database table, compute path, or deployment surface has diff --git a/docs/engineering/skills/README.md b/docs/engineering/skills/README.md index a8d6708ed..8d641a20d 100644 --- a/docs/engineering/skills/README.md +++ b/docs/engineering/skills/README.md @@ -18,3 +18,5 @@ Current skills: metadata, generated migration artifacts, and quality guards. - `testing.md`: focused test commands and dependency boundaries for migration work. +- `v2-code-organization.md`: mandatory API v2 route, service, and database-access + package boundaries and literal module-naming rules. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index d55fd25fc..efb82ad58 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -159,9 +159,9 @@ that result in the handoff instead of hiding it. ## Phase 10 Policy Migration Run the configured static type check for the Phase 10 v2 query, route, -application-service, metadata-read-repository, policy-repository, and -association-repository modules. The configured file set deliberately excludes -the existing v1 implementation: +application-service, metadata-query, policy-query and persistence, and +association-query and persistence modules. The configured file set deliberately +excludes the existing v1 implementation: ```bash uv run --frozen --extra dev mypy diff --git a/docs/engineering/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md similarity index 52% rename from docs/engineering/v2-code-organization.md rename to docs/engineering/skills/v2-code-organization.md index ded98785a..99ee8b23a 100644 --- a/docs/engineering/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -1,7 +1,7 @@ # API v2 Code Organization API v2 resource code is divided by both resource and responsibility. Public -HTTP behavior must not be implemented in database repositories, and database +HTTP behavior must not be implemented in database-access modules, and database sessions and transactions must not be opened by route modules. ## HTTP adapters @@ -58,33 +58,42 @@ SQL reads and writes live under `policyengine_api/data/v2/`: ```text policies/ - read_repository.py - write_repository.py - catalog_repository.py - legacy_mapping_repository.py + queries.py + persistence.py + catalog_resolution.py + legacy_mappings.py canonicalization.py user_policies/ - read_repository.py - write_repository.py - legacy_mapping_repository.py + queries.py + persistence.py + legacy_mappings.py metadata/ read_models.py - read_repository.py - *_read_repository.py + query_support.py + *_queries.py ``` -Read repositories execute selections and return framework-neutral read models. -Write repositories mutate SQLModel rows using a caller-provided session. -Legacy mapping repositories contain durable identity-mapping SQL and conflict -handling. The shared `data/v2/catalog/` package remains responsible for catalog -initialization, publication, and catalog selection used by multiple resources. +Query modules execute selections and return framework-neutral read models. +Persistence modules insert, update, or delete SQLModel rows using a +caller-provided session. Catalog-resolution modules select and validate the +exact catalog records needed by a resource. Legacy-mapping modules contain the +SQL and conflict handling for durable legacy-ID mappings. The shared +`data/v2/catalog/` package remains responsible for catalog initialization, +publication, and catalog selection used by multiple resources. + +Name a database-access module for the operation or data concern it implements. +Do not use `repository` as a generic synonym for SQL access. Reserve that term +for a deliberate Repository-pattern abstraction with a stable interface that +hides interchangeable persistence implementations. Direct SQL query and +mutation modules in API v2 do not currently provide that abstraction. The ordinary request direction is: ```text -route -> service -> repository -> SQLModel tables +route -> service -> query or persistence module -> SQLModel tables ``` -HTTP response models may consume framework-neutral repository read models. -Repository write functions may consume immutable application command models, -but they must not import route modules or construct HTTP responses. +HTTP response models may consume framework-neutral database read models. +Persistence functions may consume immutable application command models, but +database-access modules must not import route modules or construct HTTP +responses. diff --git a/policyengine_api/data/v2/metadata/__init__.py b/policyengine_api/data/v2/metadata/__init__.py index b92e3f7de..041bb53ce 100644 --- a/policyengine_api/data/v2/metadata/__init__.py +++ b/policyengine_api/data/v2/metadata/__init__.py @@ -1 +1 @@ -"""Read models and database repositories for API v2 metadata.""" +"""Read models and database queries for API v2 metadata.""" diff --git a/policyengine_api/data/v2/metadata/dataset_read_repository.py b/policyengine_api/data/v2/metadata/dataset_queries.py similarity index 94% rename from policyengine_api/data/v2/metadata/dataset_read_repository.py rename to policyengine_api/data/v2/metadata/dataset_queries.py index f0bf9e337..babba6937 100644 --- a/policyengine_api/data/v2/metadata/dataset_read_repository.py +++ b/policyengine_api/data/v2/metadata/dataset_queries.py @@ -5,8 +5,8 @@ from uuid import UUID from sqlmodel import col, select -from policyengine_api.data.v2.metadata.read_repository import ( - MetadataReadRepositoryBase, +from policyengine_api.data.v2.metadata.query_support import ( + MetadataQueryContext, MetadataResourceNotFoundError, page_result, query_rows, @@ -28,7 +28,7 @@ def _dataset(dataset: Dataset) -> MetadataDataset: ) -class DatasetReadRepository(MetadataReadRepositoryBase): +class DatasetQueryMethods(MetadataQueryContext): """Read logical input datasets from the selected catalog.""" def list_datasets( diff --git a/policyengine_api/data/v2/metadata/model_read_repository.py b/policyengine_api/data/v2/metadata/model_queries.py similarity index 96% rename from policyengine_api/data/v2/metadata/model_read_repository.py rename to policyengine_api/data/v2/metadata/model_queries.py index a4f6855d5..1a68bfb31 100644 --- a/policyengine_api/data/v2/metadata/model_read_repository.py +++ b/policyengine_api/data/v2/metadata/model_queries.py @@ -5,8 +5,8 @@ from uuid import UUID from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog -from policyengine_api.data.v2.metadata.read_repository import ( - MetadataReadRepositoryBase, +from policyengine_api.data.v2.metadata.query_support import ( + MetadataQueryContext, MetadataResourceNotFoundError, page_result, ) @@ -38,7 +38,7 @@ def _model_version(selected: SelectedCatalog) -> MetadataModelVersionDetail: ) -class ModelReadRepository(MetadataReadRepositoryBase): +class ModelQueryMethods(MetadataQueryContext): """Read tax-benefit models and model versions from the selected catalog.""" def list_models( diff --git a/policyengine_api/data/v2/metadata/parameter_read_repository.py b/policyengine_api/data/v2/metadata/parameter_queries.py similarity index 97% rename from policyengine_api/data/v2/metadata/parameter_read_repository.py rename to policyengine_api/data/v2/metadata/parameter_queries.py index 90367bb03..345ea0bf2 100644 --- a/policyengine_api/data/v2/metadata/parameter_read_repository.py +++ b/policyengine_api/data/v2/metadata/parameter_queries.py @@ -7,12 +7,12 @@ import sqlalchemy as sa from sqlmodel import col, select -from policyengine_api.data.v2.metadata.parameter_tree_read_repository import ( +from policyengine_api.data.v2.metadata.parameter_tree_queries import ( parameter_children_from_rows, parameter_children_query, ) -from policyengine_api.data.v2.metadata.read_repository import ( - MetadataReadRepositoryBase, +from policyengine_api.data.v2.metadata.query_support import ( + MetadataQueryContext, MetadataResourceNotFoundError, escape_like, page_result, @@ -60,7 +60,7 @@ def _utc_day_start(selected_time: datetime) -> datetime: ) -class ParameterReadRepository(MetadataReadRepositoryBase): +class ParameterQueryMethods(MetadataQueryContext): """Read parameters and canonical values from the selected catalog.""" def list_parameters( diff --git a/policyengine_api/data/v2/metadata/parameter_tree_read_repository.py b/policyengine_api/data/v2/metadata/parameter_tree_queries.py similarity index 98% rename from policyengine_api/data/v2/metadata/parameter_tree_read_repository.py rename to policyengine_api/data/v2/metadata/parameter_tree_queries.py index 238ae92fa..d653d000b 100644 --- a/policyengine_api/data/v2/metadata/parameter_tree_read_repository.py +++ b/policyengine_api/data/v2/metadata/parameter_tree_queries.py @@ -1,4 +1,4 @@ -"""Database reads for direct parameter-tree children.""" +"""Database queries for direct parameter-tree children.""" from __future__ import annotations diff --git a/policyengine_api/data/v2/metadata/read_repository.py b/policyengine_api/data/v2/metadata/query_support.py similarity index 96% rename from policyengine_api/data/v2/metadata/read_repository.py rename to policyengine_api/data/v2/metadata/query_support.py index 3bc424d24..23d6d3d8d 100644 --- a/policyengine_api/data/v2/metadata/read_repository.py +++ b/policyengine_api/data/v2/metadata/query_support.py @@ -1,4 +1,4 @@ -"""Shared database execution and pagination for v2 metadata reads.""" +"""Shared database execution and pagination for v2 metadata queries.""" from __future__ import annotations @@ -27,8 +27,8 @@ class InvalidMetadataPageError(ValueError): ResourceT = TypeVar("ResourceT") -class MetadataReadRepositoryBase: - """Own the session and catalog selection shared by metadata repositories.""" +class MetadataQueryContext: + """Own the session and catalog selection shared by metadata query methods.""" def __init__(self, session: Session, *, running_policyengine_version: str): self._session = session diff --git a/policyengine_api/data/v2/metadata/region_read_repository.py b/policyengine_api/data/v2/metadata/region_queries.py similarity index 97% rename from policyengine_api/data/v2/metadata/region_read_repository.py rename to policyengine_api/data/v2/metadata/region_queries.py index 2f574f915..67b5e3df6 100644 --- a/policyengine_api/data/v2/metadata/region_read_repository.py +++ b/policyengine_api/data/v2/metadata/region_queries.py @@ -10,8 +10,8 @@ from policyengine_api.data.v2.catalog.catalog_selection import ( MetadataCatalogUnavailableError, ) -from policyengine_api.data.v2.metadata.read_repository import ( - MetadataReadRepositoryBase, +from policyengine_api.data.v2.metadata.query_support import ( + MetadataQueryContext, MetadataResourceNotFoundError, page_result, query_rows, @@ -46,7 +46,7 @@ def _region(region: Region) -> MetadataRegion: ) -class RegionReadRepository(MetadataReadRepositoryBase): +class RegionQueryMethods(MetadataQueryContext): """Read regions and economy options from the selected catalog.""" def list_regions( diff --git a/policyengine_api/data/v2/metadata/variable_read_repository.py b/policyengine_api/data/v2/metadata/variable_queries.py similarity index 95% rename from policyengine_api/data/v2/metadata/variable_read_repository.py rename to policyengine_api/data/v2/metadata/variable_queries.py index b4f51f659..7bbb4bb0d 100644 --- a/policyengine_api/data/v2/metadata/variable_read_repository.py +++ b/policyengine_api/data/v2/metadata/variable_queries.py @@ -6,8 +6,8 @@ import sqlalchemy as sa from sqlmodel import col, select -from policyengine_api.data.v2.metadata.read_repository import ( - MetadataReadRepositoryBase, +from policyengine_api.data.v2.metadata.query_support import ( + MetadataQueryContext, MetadataResourceNotFoundError, escape_like, page_result, @@ -36,7 +36,7 @@ def _variable(variable: Variable) -> MetadataVariable: ) -class VariableReadRepository(MetadataReadRepositoryBase): +class VariableQueryMethods(MetadataQueryContext): """Read variables from the selected catalog.""" def list_variables( diff --git a/policyengine_api/data/v2/policies/catalog_repository.py b/policyengine_api/data/v2/policies/catalog_resolution.py similarity index 97% rename from policyengine_api/data/v2/policies/catalog_repository.py rename to policyengine_api/data/v2/policies/catalog_resolution.py index f99aece8e..c8e37862d 100644 --- a/policyengine_api/data/v2/policies/catalog_repository.py +++ b/policyengine_api/data/v2/policies/catalog_resolution.py @@ -1,4 +1,4 @@ -"""Catalog database validation for immutable v2 policy commands.""" +"""Catalog resolution and validation for immutable v2 policy commands.""" from __future__ import annotations diff --git a/policyengine_api/data/v2/policies/legacy_mapping_repository.py b/policyengine_api/data/v2/policies/legacy_mappings.py similarity index 96% rename from policyengine_api/data/v2/policies/legacy_mapping_repository.py rename to policyengine_api/data/v2/policies/legacy_mappings.py index 3f1ec3f4a..0b5305316 100644 --- a/policyengine_api/data/v2/policies/legacy_mapping_repository.py +++ b/policyengine_api/data/v2/policies/legacy_mappings.py @@ -1,4 +1,4 @@ -"""Database operations for durable v1-policy-to-v2-policy mappings.""" +"""SQL operations for durable v1-policy-to-v2-policy mappings.""" from __future__ import annotations diff --git a/policyengine_api/data/v2/policies/write_repository.py b/policyengine_api/data/v2/policies/persistence.py similarity index 98% rename from policyengine_api/data/v2/policies/write_repository.py rename to policyengine_api/data/v2/policies/persistence.py index b839f8b83..8815f2dfa 100644 --- a/policyengine_api/data/v2/policies/write_repository.py +++ b/policyengine_api/data/v2/policies/persistence.py @@ -1,4 +1,4 @@ -"""Conflict-aware PostgreSQL writes for immutable v2 policies.""" +"""Conflict-aware PostgreSQL persistence for immutable v2 policies.""" from __future__ import annotations diff --git a/policyengine_api/data/v2/policies/read_repository.py b/policyengine_api/data/v2/policies/queries.py similarity index 98% rename from policyengine_api/data/v2/policies/read_repository.py rename to policyengine_api/data/v2/policies/queries.py index adb2cfab0..6f8c22551 100644 --- a/policyengine_api/data/v2/policies/read_repository.py +++ b/policyengine_api/data/v2/policies/queries.py @@ -1,4 +1,4 @@ -"""Country-scoped database reads for immutable v2 policies.""" +"""Country-scoped database queries for immutable v2 policies.""" from __future__ import annotations diff --git a/policyengine_api/data/v2/user_policies/legacy_mapping_repository.py b/policyengine_api/data/v2/user_policies/legacy_mappings.py similarity index 99% rename from policyengine_api/data/v2/user_policies/legacy_mapping_repository.py rename to policyengine_api/data/v2/user_policies/legacy_mappings.py index df0433838..51c0886fd 100644 --- a/policyengine_api/data/v2/user_policies/legacy_mapping_repository.py +++ b/policyengine_api/data/v2/user_policies/legacy_mappings.py @@ -1,4 +1,4 @@ -"""Database operations for legacy users and saved-policy association mappings.""" +"""SQL operations for legacy users and saved-policy association mappings.""" from __future__ import annotations diff --git a/policyengine_api/data/v2/user_policies/write_repository.py b/policyengine_api/data/v2/user_policies/persistence.py similarity index 95% rename from policyengine_api/data/v2/user_policies/write_repository.py rename to policyengine_api/data/v2/user_policies/persistence.py index 4afa638fa..13929bfb2 100644 --- a/policyengine_api/data/v2/user_policies/write_repository.py +++ b/policyengine_api/data/v2/user_policies/persistence.py @@ -1,4 +1,4 @@ -"""Transactional database writes for mutable user-policy associations.""" +"""Transactional database persistence for mutable user-policy associations.""" from __future__ import annotations @@ -8,7 +8,7 @@ from policyengine_api.data.v2.models import Policy, User, UserPolicy from policyengine_api.data.v2.models.base import utc_now -from policyengine_api.data.v2.user_policies.read_repository import ( +from policyengine_api.data.v2.user_policies.queries import ( UserPolicyRead, association_read, get_user_policy_row, diff --git a/policyengine_api/data/v2/user_policies/read_repository.py b/policyengine_api/data/v2/user_policies/queries.py similarity index 97% rename from policyengine_api/data/v2/user_policies/read_repository.py rename to policyengine_api/data/v2/user_policies/queries.py index 4deccb394..b183c1ad7 100644 --- a/policyengine_api/data/v2/user_policies/read_repository.py +++ b/policyengine_api/data/v2/user_policies/queries.py @@ -1,4 +1,4 @@ -"""Country-scoped database reads for v2 user-policy associations.""" +"""Country-scoped database queries for v2 user-policy associations.""" from __future__ import annotations diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 5ca4230ce..1a0028f9b 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -27,8 +27,8 @@ MetadataRegion, MetadataVariable, ) - from policyengine_api.data.v2.policies.read_repository import PolicyPage, PolicyRead - from policyengine_api.data.v2.user_policies.read_repository import ( + from policyengine_api.data.v2.policies.queries import PolicyPage, PolicyRead + from policyengine_api.data.v2.user_policies.queries import ( UserPolicyPage, UserPolicyRead, ) diff --git a/policyengine_api/fastapi_routes/v2/policies/response_models.py b/policyengine_api/fastapi_routes/v2/policies/response_models.py index 6bc31a6b0..b75db5883 100644 --- a/policyengine_api/fastapi_routes/v2/policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/policies/response_models.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue, StringConstraints -from policyengine_api.data.v2.policies.read_repository import PolicyPage, PolicyRead +from policyengine_api.data.v2.policies.queries import PolicyPage, PolicyRead class StrictPolicyAPIModel(BaseModel): diff --git a/policyengine_api/fastapi_routes/v2/policies/routes.py b/policyengine_api/fastapi_routes/v2/policies/routes.py index c8d8cfe34..a9195a536 100644 --- a/policyengine_api/fastapi_routes/v2/policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/policies/routes.py @@ -27,14 +27,14 @@ PolicyPageResponse, PolicyPageResult, ) -from policyengine_api.data.v2.policies.catalog_repository import ( +from policyengine_api.data.v2.policies.catalog_resolution import ( PolicyCatalogValidationError, ) -from policyengine_api.data.v2.policies.read_repository import PolicyNotFoundError -from policyengine_api.data.v2.policies.write_repository import ( +from policyengine_api.data.v2.policies.persistence import ( PolicyContentHashCollisionError, PolicyPersistenceIntegrityError, ) +from policyengine_api.data.v2.policies.queries import PolicyNotFoundError from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import ( NativeRouteDependencies, diff --git a/policyengine_api/fastapi_routes/v2/user_policies/response_models.py b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py index eb18fa8f3..e82db0dd9 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, StringConstraints -from policyengine_api.data.v2.user_policies.read_repository import ( +from policyengine_api.data.v2.user_policies.queries import ( UserPolicyPage, UserPolicyRead, ) diff --git a/policyengine_api/fastapi_routes/v2/user_policies/routes.py b/policyengine_api/fastapi_routes/v2/user_policies/routes.py index 5725023ea..bf13a726d 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/routes.py @@ -24,10 +24,10 @@ UserPolicyPageResponse, UserPolicyPageResult, ) -from policyengine_api.data.v2.user_policies.read_repository import ( +from policyengine_api.data.v2.user_policies.queries import ( UserPolicyNotFoundError, ) -from policyengine_api.data.v2.user_policies.write_repository import ( +from policyengine_api.data.v2.user_policies.persistence import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py index 48432b523..151d85d78 100644 --- a/policyengine_api/services/policy_mirroring.py +++ b/policyengine_api/services/policy_mirroring.py @@ -12,13 +12,13 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.catalog_repository import ( +from policyengine_api.data.v2.policies.catalog_resolution import ( PolicyCatalogValidationError, ) -from policyengine_api.data.v2.policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.policies.legacy_mappings import ( LegacyPolicyMappingIntegrityError, ) -from policyengine_api.data.v2.policies.write_repository import ( +from policyengine_api.data.v2.policies.persistence import ( PolicyContentHashCollisionError, PolicyPersistenceIntegrityError, ) diff --git a/policyengine_api/services/user_policy_mirroring.py b/policyengine_api/services/user_policy_mirroring.py index e28e39f95..0e0efd1fa 100644 --- a/policyengine_api/services/user_policy_mirroring.py +++ b/policyengine_api/services/user_policy_mirroring.py @@ -12,17 +12,17 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.catalog_repository import ( +from policyengine_api.data.v2.policies.catalog_resolution import ( PolicyCatalogValidationError, ) -from policyengine_api.data.v2.policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.policies.legacy_mappings import ( LegacyPolicyMappingIntegrityError, ) -from policyengine_api.data.v2.policies.write_repository import ( +from policyengine_api.data.v2.policies.persistence import ( PolicyContentHashCollisionError, PolicyPersistenceIntegrityError, ) -from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.user_policies.legacy_mappings import ( LegacyUserPolicyIntegrityError, LegacyUserPolicyPersistenceResult, ) diff --git a/policyengine_api/services/user_policy_service.py b/policyengine_api/services/user_policy_service.py index d175381fe..5f5782884 100644 --- a/policyengine_api/services/user_policy_service.py +++ b/policyengine_api/services/user_policy_service.py @@ -17,7 +17,7 @@ from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent -from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.user_policies.legacy_mappings import ( LegacyUserPolicyPersistenceResult, ) from policyengine_api.services.v2.user_policies.legacy_translation import ( diff --git a/policyengine_api/services/v2/metadata/service.py b/policyengine_api/services/v2/metadata/service.py index 36f29ae94..88420c098 100644 --- a/policyengine_api/services/v2/metadata/service.py +++ b/policyengine_api/services/v2/metadata/service.py @@ -9,25 +9,25 @@ UnsupportedPreviewCountryError, validate_policyengine_version, ) -from policyengine_api.data.v2.metadata.dataset_read_repository import ( - DatasetReadRepository, +from policyengine_api.data.v2.metadata.dataset_queries import ( + DatasetQueryMethods, ) -from policyengine_api.data.v2.metadata.model_read_repository import ( - ModelReadRepository, +from policyengine_api.data.v2.metadata.model_queries import ( + ModelQueryMethods, ) -from policyengine_api.data.v2.metadata.parameter_read_repository import ( - ParameterReadRepository, +from policyengine_api.data.v2.metadata.parameter_queries import ( + ParameterQueryMethods, ) -from policyengine_api.data.v2.metadata.read_repository import ( +from policyengine_api.data.v2.metadata.query_support import ( InvalidMetadataPageError, MetadataResourceNotFoundError, validate_metadata_page, ) -from policyengine_api.data.v2.metadata.region_read_repository import ( - RegionReadRepository, +from policyengine_api.data.v2.metadata.region_queries import ( + RegionQueryMethods, ) -from policyengine_api.data.v2.metadata.variable_read_repository import ( - VariableReadRepository, +from policyengine_api.data.v2.metadata.variable_queries import ( + VariableQueryMethods, ) @@ -45,10 +45,10 @@ class V2MetadataService( - ModelReadRepository, - VariableReadRepository, - ParameterReadRepository, - DatasetReadRepository, - RegionReadRepository, + ModelQueryMethods, + VariableQueryMethods, + ParameterQueryMethods, + DatasetQueryMethods, + RegionQueryMethods, ): - """Expose resource-specific metadata repositories to the route layer.""" + """Expose resource-specific metadata query methods to the route layer.""" diff --git a/policyengine_api/services/v2/policies/legacy_service.py b/policyengine_api/services/v2/policies/legacy_service.py index ad66e079d..633b648f7 100644 --- a/policyengine_api/services/v2/policies/legacy_service.py +++ b/policyengine_api/services/v2/policies/legacy_service.py @@ -9,13 +9,13 @@ from sqlmodel import Session from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION -from policyengine_api.data.v2.policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.policies.legacy_mappings import ( LegacyPolicyMappingIntegrityError, find_legacy_policy_mapping, insert_legacy_policy_mapping, verify_legacy_policy_mapping, ) -from policyengine_api.data.v2.policies.write_repository import ( +from policyengine_api.data.v2.policies.persistence import ( persist_resolved_policy, ) from policyengine_api.services.v2.policies.legacy_translation import ( diff --git a/policyengine_api/services/v2/policies/legacy_translation.py b/policyengine_api/services/v2/policies/legacy_translation.py index 1f6830aa9..190be22eb 100644 --- a/policyengine_api/services/v2/policies/legacy_translation.py +++ b/policyengine_api/services/v2/policies/legacy_translation.py @@ -16,7 +16,7 @@ from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION from policyengine_api.data.v2.catalog.catalog_selection import select_catalog from policyengine_api.data.v2.models import Parameter -from policyengine_api.data.v2.policies.catalog_repository import ( +from policyengine_api.data.v2.policies.catalog_resolution import ( resolve_policy_catalog, ) from policyengine_api.query_parameters import CountryId diff --git a/policyengine_api/services/v2/policies/service.py b/policyengine_api/services/v2/policies/service.py index 7a6604ee9..1dddd2d64 100644 --- a/policyengine_api/services/v2/policies/service.py +++ b/policyengine_api/services/v2/policies/service.py @@ -9,14 +9,14 @@ from sqlmodel import Session from policyengine_api.constants import POLICYENGINE_VERSION -from policyengine_api.data.v2.policies.catalog_repository import resolve_policy_catalog -from policyengine_api.data.v2.policies.read_repository import ( +from policyengine_api.data.v2.policies.catalog_resolution import resolve_policy_catalog +from policyengine_api.data.v2.policies.persistence import persist_resolved_policy +from policyengine_api.data.v2.policies.queries import ( PolicyPage, PolicyRead, list_policies, read_policy, ) -from policyengine_api.data.v2.policies.write_repository import persist_resolved_policy from policyengine_api.services.v2.policies.commands import ( NativePolicyCreateCommand, PolicyCreateCommand, diff --git a/policyengine_api/services/v2/user_policies/legacy_service.py b/policyengine_api/services/v2/user_policies/legacy_service.py index 1470e7666..9fe8673be 100644 --- a/policyengine_api/services/v2/user_policies/legacy_service.py +++ b/policyengine_api/services/v2/user_policies/legacy_service.py @@ -4,7 +4,7 @@ from sqlmodel import Session -from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.user_policies.legacy_mappings import ( LegacyUserPolicyIntegrityError, LegacyUserPolicyPersistenceResult, persist_legacy_user_policy_mapping, diff --git a/policyengine_api/services/v2/user_policies/legacy_translation.py b/policyengine_api/services/v2/user_policies/legacy_translation.py index 7bd03b9fc..eb994e812 100644 --- a/policyengine_api/services/v2/user_policies/legacy_translation.py +++ b/policyengine_api/services/v2/user_policies/legacy_translation.py @@ -9,7 +9,7 @@ from pydantic import Field -from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.user_policies.legacy_mappings import ( USER_POLICY_FINGERPRINT_VERSION, ) from policyengine_api.query_parameters import CountryId, LegacyUserId diff --git a/policyengine_api/services/v2/user_policies/service.py b/policyengine_api/services/v2/user_policies/service.py index dfe5e85d9..b67caee55 100644 --- a/policyengine_api/services/v2/user_policies/service.py +++ b/policyengine_api/services/v2/user_policies/service.py @@ -7,13 +7,13 @@ from sqlalchemy.orm import sessionmaker from sqlmodel import Session -from policyengine_api.data.v2.user_policies.read_repository import ( +from policyengine_api.data.v2.user_policies.queries import ( UserPolicyPage, UserPolicyRead, list_user_policies, read_user_policy, ) -from policyengine_api.data.v2.user_policies.write_repository import ( +from policyengine_api.data.v2.user_policies.persistence import ( create_user_policy, delete_user_policy, patch_user_policy, diff --git a/tests/integration/test_v2_policy_persistence.py b/tests/integration/test_v2_policy_persistence.py index 2010f1c08..81705c3f8 100644 --- a/tests/integration/test_v2_policy_persistence.py +++ b/tests/integration/test_v2_policy_persistence.py @@ -29,10 +29,10 @@ canonical_policy_document, canonicalize_policy, ) -from policyengine_api.data.v2.policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.policies.legacy_mappings import ( LegacyPolicyMappingIntegrityError, ) -from policyengine_api.data.v2.policies.write_repository import ( +from policyengine_api.data.v2.policies.persistence import ( PolicyContentHashCollisionError, persist_resolved_policy, ) diff --git a/tests/integration/test_v2_user_policy_mirroring.py b/tests/integration/test_v2_user_policy_mirroring.py index 2344884d9..791e2d60e 100644 --- a/tests/integration/test_v2_user_policy_mirroring.py +++ b/tests/integration/test_v2_user_policy_mirroring.py @@ -29,7 +29,7 @@ User, UserPolicy, ) -from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.user_policies.legacy_mappings import ( resolve_legacy_user_id, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL diff --git a/tests/unit/services/test_policy_mirroring.py b/tests/unit/services/test_policy_mirroring.py index 765daf9cf..e41378832 100644 --- a/tests/unit/services/test_policy_mirroring.py +++ b/tests/unit/services/test_policy_mirroring.py @@ -8,7 +8,7 @@ import pytest from sqlalchemy.exc import OperationalError, TimeoutError -from policyengine_api.data.v2.policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.policies.legacy_mappings import ( LegacyPolicyMappingIntegrityError, ) from policyengine_api.services.v2.policies.legacy_service import ( diff --git a/tests/unit/services/test_user_policy_mirroring.py b/tests/unit/services/test_user_policy_mirroring.py index cfbf443ab..3fb0d4195 100644 --- a/tests/unit/services/test_user_policy_mirroring.py +++ b/tests/unit/services/test_user_policy_mirroring.py @@ -10,7 +10,7 @@ from sqlalchemy.exc import OperationalError, TimeoutError from policyengine_api.data.v1_models import UserPolicyMirrorEvent -from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.user_policies.legacy_mappings import ( LegacyUserPolicyIntegrityError, LegacyUserPolicyPersistenceResult, ) diff --git a/tests/unit/services/test_user_policy_service.py b/tests/unit/services/test_user_policy_service.py index 8c85700ce..aae90f3cf 100644 --- a/tests/unit/services/test_user_policy_service.py +++ b/tests/unit/services/test_user_policy_service.py @@ -14,7 +14,7 @@ ) from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent -from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.user_policies.legacy_mappings import ( LegacyUserPolicyPersistenceResult, ) from policyengine_api.services.user_policy_service import ( diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 3cd45465e..d24372120 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -551,17 +551,17 @@ def test_economy_options_require_a_national_region_and_dataset( _service(catalog_session).get_economy_options("us") -def test_read_repositories_import_no_policyengine_or_v1_metadata_source() -> None: +def test_query_modules_import_no_policyengine_or_v1_metadata_source() -> None: data_directory = Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" modules = ( data_directory / "catalog" / "catalog_selection.py", - data_directory / "metadata" / "dataset_read_repository.py", - data_directory / "metadata" / "model_read_repository.py", - data_directory / "metadata" / "parameter_read_repository.py", - data_directory / "metadata" / "parameter_tree_read_repository.py", - data_directory / "metadata" / "read_repository.py", - data_directory / "metadata" / "region_read_repository.py", - data_directory / "metadata" / "variable_read_repository.py", + data_directory / "metadata" / "dataset_queries.py", + data_directory / "metadata" / "model_queries.py", + data_directory / "metadata" / "parameter_queries.py", + data_directory / "metadata" / "parameter_tree_queries.py", + data_directory / "metadata" / "query_support.py", + data_directory / "metadata" / "region_queries.py", + data_directory / "metadata" / "variable_queries.py", ) imported = set() for module in modules: @@ -593,26 +593,26 @@ def test_read_repositories_import_no_policyengine_or_v1_metadata_source() -> Non ) -def test_resource_service_methods_are_defined_in_their_read_repositories() -> None: +def test_resource_service_methods_are_defined_in_their_query_modules() -> None: expected_modules = { - "list_models": "model_read_repository", - "get_model": "model_read_repository", - "get_model_by_country": "model_read_repository", - "list_model_versions": "model_read_repository", - "get_model_version": "model_read_repository", - "list_variables": "variable_read_repository", - "get_variable": "variable_read_repository", - "list_parameters": "parameter_read_repository", - "get_parameter": "parameter_read_repository", - "list_parameter_children": "parameter_read_repository", - "list_parameter_values": "parameter_read_repository", - "get_parameter_value": "parameter_read_repository", - "list_datasets": "dataset_read_repository", - "get_dataset": "dataset_read_repository", - "list_regions": "region_read_repository", - "get_region": "region_read_repository", - "get_region_by_code": "region_read_repository", - "get_economy_options": "region_read_repository", + "list_models": "model_queries", + "get_model": "model_queries", + "get_model_by_country": "model_queries", + "list_model_versions": "model_queries", + "get_model_version": "model_queries", + "list_variables": "variable_queries", + "get_variable": "variable_queries", + "list_parameters": "parameter_queries", + "get_parameter": "parameter_queries", + "list_parameter_children": "parameter_queries", + "list_parameter_values": "parameter_queries", + "get_parameter_value": "parameter_queries", + "list_datasets": "dataset_queries", + "get_dataset": "dataset_queries", + "list_regions": "region_queries", + "get_region": "region_queries", + "get_region_by_code": "region_queries", + "get_economy_options": "region_queries", } for method_name, module_name in expected_modules.items(): diff --git a/tests/unit/v2/test_policy_catalog.py b/tests/unit/v2/test_policy_catalog.py index db72cffbf..9d61b83ed 100644 --- a/tests/unit/v2/test_policy_catalog.py +++ b/tests/unit/v2/test_policy_catalog.py @@ -16,7 +16,7 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.data.v2.policies.catalog_repository import ( +from policyengine_api.data.v2.policies.catalog_resolution import ( PolicyCatalogValidationError, resolve_policy_catalog, ) diff --git a/tests/unit/v2/test_policy_persistence_statements.py b/tests/unit/v2/test_policy_persistence_statements.py index 5163e29f3..f51716d86 100644 --- a/tests/unit/v2/test_policy_persistence_statements.py +++ b/tests/unit/v2/test_policy_persistence_statements.py @@ -4,11 +4,11 @@ from sqlalchemy.dialects import postgresql -from policyengine_api.data.v2.policies import write_repository +from policyengine_api.data.v2.policies import persistence def test_policy_insert_uses_the_content_identity_constraint_and_returning() -> None: - source = write_repository._insert_policy.__code__.co_consts + source = persistence._insert_policy.__code__.co_consts statement_text = " ".join(str(value) for value in source) assert "uq_policies_canonicalization_content_hash" in statement_text @@ -16,9 +16,9 @@ def test_policy_insert_uses_the_content_identity_constraint_and_returning() -> N # Compile a representative statement through the same PostgreSQL dialect # construct to prove this module does not use a read-before-write insert. statement = ( - write_repository.insert(write_repository.Policy) + persistence.insert(persistence.Policy) .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") - .returning(write_repository.Policy.id) + .returning(persistence.Policy.id) ) compiled = str(statement.compile(dialect=postgresql.dialect())) assert "ON CONFLICT ON CONSTRAINT" in compiled diff --git a/tests/unit/v2/test_policy_query.py b/tests/unit/v2/test_policy_query.py index 2f2616e69..ed5fa15ab 100644 --- a/tests/unit/v2/test_policy_query.py +++ b/tests/unit/v2/test_policy_query.py @@ -16,7 +16,7 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.data.v2.policies.read_repository import ( +from policyengine_api.data.v2.policies.queries import ( PolicyNotFoundError, list_policies, read_policy, diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py index 2b2a0da4b..7890e2e5f 100644 --- a/tests/unit/v2/test_policy_routes.py +++ b/tests/unit/v2/test_policy_routes.py @@ -15,16 +15,16 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.catalog_repository import ( +from policyengine_api.data.v2.policies.catalog_resolution import ( PolicyCatalogValidationError, ) -from policyengine_api.data.v2.policies.read_repository import ( +from policyengine_api.data.v2.policies.queries import ( PolicyNotFoundError, PolicyPage, PolicyParameterValueRead, PolicyRead, ) -from policyengine_api.data.v2.policies.write_repository import ( +from policyengine_api.data.v2.policies.persistence import ( PolicyContentHashCollisionError, PolicyPersistenceIntegrityError, ) diff --git a/tests/unit/v2/test_user_policy_legacy.py b/tests/unit/v2/test_user_policy_legacy.py index 69d96293d..bcf78d079 100644 --- a/tests/unit/v2/test_user_policy_legacy.py +++ b/tests/unit/v2/test_user_policy_legacy.py @@ -8,7 +8,7 @@ import pytest from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy -from policyengine_api.data.v2.user_policies.legacy_mapping_repository import ( +from policyengine_api.data.v2.user_policies.legacy_mappings import ( LegacyUserPolicyIntegrityError, apply_existing_legacy_user_policy_mapping, ) diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py index 660c3e76b..f24720adb 100644 --- a/tests/unit/v2/test_user_policy_routes.py +++ b/tests/unit/v2/test_user_policy_routes.py @@ -12,12 +12,12 @@ from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.data.v2.settings import V2ConfigurationError -from policyengine_api.data.v2.user_policies.read_repository import ( +from policyengine_api.data.v2.user_policies.queries import ( UserPolicyNotFoundError, UserPolicyPage, UserPolicyRead, ) -from policyengine_api.data.v2.user_policies.write_repository import ( +from policyengine_api.data.v2.user_policies.persistence import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py index fdd37bbb9..40b432bb5 100644 --- a/tests/unit/v2/test_user_policy_service.py +++ b/tests/unit/v2/test_user_policy_service.py @@ -21,10 +21,10 @@ UserPolicy, V2_METADATA, ) -from policyengine_api.data.v2.user_policies.read_repository import ( +from policyengine_api.data.v2.user_policies.queries import ( UserPolicyNotFoundError, ) -from policyengine_api.data.v2.user_policies.write_repository import ( +from policyengine_api.data.v2.user_policies.persistence import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, From 93386cf489aaa4441668ee1a3734c62d4e60a866 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:14:29 +0400 Subject: [PATCH 09/18] Separate API v2 database operations by CRUD action --- docs/engineering/skills/testing.md | 7 +- .../skills/v2-code-organization.md | 64 ++-- policyengine_api/data/v2/metadata/__init__.py | 2 +- .../{dataset_queries.py => dataset_reads.py} | 6 +- .../{model_queries.py => model_reads.py} | 6 +- ...arameter_queries.py => parameter_reads.py} | 8 +- ...ree_queries.py => parameter_tree_reads.py} | 2 +- .../{query_support.py => read_support.py} | 6 +- .../{region_queries.py => region_reads.py} | 6 +- ...{variable_queries.py => variable_reads.py} | 6 +- policyengine_api/data/v2/policies/__init__.py | 2 +- .../data/v2/policies/catalog_resolution.py | 77 ----- policyengine_api/data/v2/policies/creates.py | 86 ++++++ .../data/v2/policies/legacy_mappings.py | 69 ----- .../data/v2/policies/persistence.py | 162 ---------- .../data/v2/policies/{queries.py => reads.py} | 140 ++++++++- .../data/v2/user_policies/__init__.py | 2 +- .../data/v2/user_policies/creates.py | 89 ++++++ .../data/v2/user_policies/deletes.py | 21 ++ .../data/v2/user_policies/legacy_mappings.py | 278 ------------------ .../data/v2/user_policies/persistence.py | 99 ------- .../v2/user_policies/{queries.py => reads.py} | 70 ++++- .../data/v2/user_policies/updates.py | 48 +++ .../fastapi_routes/dependencies.py | 4 +- .../v2/policies/response_models.py | 2 +- .../fastapi_routes/v2/policies/routes.py | 10 +- .../v2/user_policies/response_models.py | 2 +- .../fastapi_routes/v2/user_policies/routes.py | 4 +- policyengine_api/services/policy_mirroring.py | 14 +- .../services/user_policy_mirroring.py | 16 +- .../services/user_policy_service.py | 2 +- .../services/v2/metadata/service.py | 34 +-- .../v2/policies/canonicalization.py | 2 +- .../v2/policies/catalog_validation.py | 43 +++ .../services/v2/policies/creation.py | 123 ++++++++ .../services/v2/policies/legacy_service.py | 45 ++- .../v2/policies/legacy_translation.py | 42 +-- .../services/v2/policies/service.py | 10 +- .../v2/user_policies/legacy_service.py | 230 ++++++++++++++- .../v2/user_policies/legacy_translation.py | 6 +- .../services/v2/user_policies/service.py | 52 +++- .../integration/test_v2_policy_persistence.py | 26 +- .../test_v2_user_policy_mirroring.py | 2 +- tests/unit/services/test_policy_mirroring.py | 2 +- .../services/test_user_policy_mirroring.py | 2 +- .../unit/services/test_user_policy_service.py | 2 +- tests/unit/v2/test_data_crud_boundaries.py | 89 ++++++ tests/unit/v2/test_metadata_service.py | 54 ++-- tests/unit/v2/test_policy_canonicalization.py | 2 +- tests/unit/v2/test_policy_catalog.py | 4 +- .../v2/test_policy_persistence_statements.py | 8 +- tests/unit/v2/test_policy_query.py | 2 +- tests/unit/v2/test_policy_routes.py | 14 +- tests/unit/v2/test_user_policy_legacy.py | 2 +- tests/unit/v2/test_user_policy_routes.py | 4 +- tests/unit/v2/test_user_policy_service.py | 4 +- 56 files changed, 1207 insertions(+), 907 deletions(-) rename policyengine_api/data/v2/metadata/{dataset_queries.py => dataset_reads.py} (94%) rename policyengine_api/data/v2/metadata/{model_queries.py => model_reads.py} (96%) rename policyengine_api/data/v2/metadata/{parameter_queries.py => parameter_reads.py} (97%) rename policyengine_api/data/v2/metadata/{parameter_tree_queries.py => parameter_tree_reads.py} (98%) rename policyengine_api/data/v2/metadata/{query_support.py => read_support.py} (96%) rename policyengine_api/data/v2/metadata/{region_queries.py => region_reads.py} (97%) rename policyengine_api/data/v2/metadata/{variable_queries.py => variable_reads.py} (95%) delete mode 100644 policyengine_api/data/v2/policies/catalog_resolution.py create mode 100644 policyengine_api/data/v2/policies/creates.py delete mode 100644 policyengine_api/data/v2/policies/legacy_mappings.py delete mode 100644 policyengine_api/data/v2/policies/persistence.py rename policyengine_api/data/v2/policies/{queries.py => reads.py} (50%) create mode 100644 policyengine_api/data/v2/user_policies/creates.py create mode 100644 policyengine_api/data/v2/user_policies/deletes.py delete mode 100644 policyengine_api/data/v2/user_policies/legacy_mappings.py delete mode 100644 policyengine_api/data/v2/user_policies/persistence.py rename policyengine_api/data/v2/user_policies/{queries.py => reads.py} (58%) create mode 100644 policyengine_api/data/v2/user_policies/updates.py rename policyengine_api/{data => services}/v2/policies/canonicalization.py (97%) create mode 100644 policyengine_api/services/v2/policies/catalog_validation.py create mode 100644 policyengine_api/services/v2/policies/creation.py create mode 100644 tests/unit/v2/test_data_crud_boundaries.py diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index efb82ad58..2bc9f2145 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -159,9 +159,9 @@ that result in the handoff instead of hiding it. ## Phase 10 Policy Migration Run the configured static type check for the Phase 10 v2 query, route, -application-service, metadata-query, policy-query and persistence, and -association-query and persistence modules. The configured file set deliberately -excludes the existing v1 implementation: +application-service, metadata-read, policy-create and read, and association +CRUD modules. The configured file set deliberately excludes the existing v1 +implementation: ```bash uv run --frozen --extra dev mypy @@ -175,6 +175,7 @@ uv run pytest \ tests/unit/test_query_parameters.py \ tests/unit/v2/test_models.py \ tests/unit/v2/test_model_persistence.py \ + tests/unit/v2/test_data_crud_boundaries.py \ tests/unit/v2/test_policy_routes.py \ tests/unit/v2/test_user_policy_routes.py \ tests/unit/v2/test_user_policy_service.py \ diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index 99ee8b23a..273ca54bb 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -35,6 +35,9 @@ Resource-specific application code lives under `policyengine_api/services/v2/`: ```text policies/ commands.py + catalog_validation.py + canonicalization.py + creation.py legacy_translation.py legacy_service.py service.py @@ -50,7 +53,10 @@ metadata/ Command models are independent of FastAPI and Flask. Native services own request-level database sessions and transaction boundaries. Legacy translation converts committed v1 snapshots into v2 commands. Legacy services coordinate -all work that must occur inside one Supabase transaction. +all work that must occur inside one Supabase transaction. Catalog validation +operates only on already-loaded records and must not execute SQL. Policy +canonicalization is deterministic application logic and must not access a +database. ## Database access @@ -58,42 +64,54 @@ SQL reads and writes live under `policyengine_api/data/v2/`: ```text policies/ - queries.py - persistence.py - catalog_resolution.py - legacy_mappings.py - canonicalization.py + creates.py + reads.py user_policies/ - queries.py - persistence.py - legacy_mappings.py + creates.py + reads.py + updates.py + deletes.py metadata/ read_models.py - query_support.py - *_queries.py + read_support.py + *_reads.py ``` -Query modules execute selections and return framework-neutral read models. -Persistence modules insert, update, or delete SQLModel rows using a -caller-provided session. Catalog-resolution modules select and validate the -exact catalog records needed by a resource. Legacy-mapping modules contain the -SQL and conflict handling for durable legacy-ID mappings. The shared +Database-access modules are organized by SQL operation rather than by HTTP +method or table. Read modules contain every `SELECT` and `Session.get` +operation used by the resource, including reads performed while processing a +create, update, or delete request. Create modules contain inserts and ORM row +creation. Update modules modify existing rows. Delete modules remove rows. A +request may use several CRUD modules while the application service sequences +those calls and owns the transaction. + +Do not create an empty CRUD module for an operation the resource does not +support. Immutable policies therefore have only `creates.py` and `reads.py`. +Mutable user-policy associations have all four modules. Read-only metadata +uses resource-specific `*_reads.py` modules and shared `read_support.py`. + +Legacy mapping SQL follows the same division: mapping selection belongs in +`reads.py`, mapping insertion in `creates.py`, mapping mutation in `updates.py`, +and mapping removal in `deletes.py`. Mapping validation and retry sequencing +belong in application services, not database-access modules. The shared `data/v2/catalog/` package remains responsible for catalog initialization, publication, and catalog selection used by multiple resources. -Name a database-access module for the operation or data concern it implements. +Name a database-access module for the CRUD operation it implements. Do not use `repository` as a generic synonym for SQL access. Reserve that term for a deliberate Repository-pattern abstraction with a stable interface that -hides interchangeable persistence implementations. Direct SQL query and -mutation modules in API v2 do not currently provide that abstraction. +hides interchangeable persistence implementations. Direct SQL CRUD modules in +API v2 do not currently provide that abstraction. The ordinary request direction is: ```text -route -> service -> query or persistence module -> SQLModel tables +route -> service -> one or more CRUD modules -> SQLModel tables ``` HTTP response models may consume framework-neutral database read models. -Persistence functions may consume immutable application command models, but -database-access modules must not import route modules or construct HTTP -responses. +CRUD functions may consume immutable application command models, but +database-access modules must not import route modules, perform request-level +validation, control the transaction, or construct HTTP responses. CRUD modules +must not call one another; the application service makes their ordering and +shared transaction explicit. diff --git a/policyengine_api/data/v2/metadata/__init__.py b/policyengine_api/data/v2/metadata/__init__.py index 041bb53ce..16af473a3 100644 --- a/policyengine_api/data/v2/metadata/__init__.py +++ b/policyengine_api/data/v2/metadata/__init__.py @@ -1 +1 @@ -"""Read models and database queries for API v2 metadata.""" +"""Read models and database reads for API v2 metadata.""" diff --git a/policyengine_api/data/v2/metadata/dataset_queries.py b/policyengine_api/data/v2/metadata/dataset_reads.py similarity index 94% rename from policyengine_api/data/v2/metadata/dataset_queries.py rename to policyengine_api/data/v2/metadata/dataset_reads.py index babba6937..b55fab08c 100644 --- a/policyengine_api/data/v2/metadata/dataset_queries.py +++ b/policyengine_api/data/v2/metadata/dataset_reads.py @@ -5,8 +5,8 @@ from uuid import UUID from sqlmodel import col, select -from policyengine_api.data.v2.metadata.query_support import ( - MetadataQueryContext, +from policyengine_api.data.v2.metadata.read_support import ( + MetadataReadContext, MetadataResourceNotFoundError, page_result, query_rows, @@ -28,7 +28,7 @@ def _dataset(dataset: Dataset) -> MetadataDataset: ) -class DatasetQueryMethods(MetadataQueryContext): +class DatasetReadMethods(MetadataReadContext): """Read logical input datasets from the selected catalog.""" def list_datasets( diff --git a/policyengine_api/data/v2/metadata/model_queries.py b/policyengine_api/data/v2/metadata/model_reads.py similarity index 96% rename from policyengine_api/data/v2/metadata/model_queries.py rename to policyengine_api/data/v2/metadata/model_reads.py index 1a68bfb31..85622b986 100644 --- a/policyengine_api/data/v2/metadata/model_queries.py +++ b/policyengine_api/data/v2/metadata/model_reads.py @@ -5,8 +5,8 @@ from uuid import UUID from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog -from policyengine_api.data.v2.metadata.query_support import ( - MetadataQueryContext, +from policyengine_api.data.v2.metadata.read_support import ( + MetadataReadContext, MetadataResourceNotFoundError, page_result, ) @@ -38,7 +38,7 @@ def _model_version(selected: SelectedCatalog) -> MetadataModelVersionDetail: ) -class ModelQueryMethods(MetadataQueryContext): +class ModelReadMethods(MetadataReadContext): """Read tax-benefit models and model versions from the selected catalog.""" def list_models( diff --git a/policyengine_api/data/v2/metadata/parameter_queries.py b/policyengine_api/data/v2/metadata/parameter_reads.py similarity index 97% rename from policyengine_api/data/v2/metadata/parameter_queries.py rename to policyengine_api/data/v2/metadata/parameter_reads.py index 345ea0bf2..9be07114e 100644 --- a/policyengine_api/data/v2/metadata/parameter_queries.py +++ b/policyengine_api/data/v2/metadata/parameter_reads.py @@ -7,12 +7,12 @@ import sqlalchemy as sa from sqlmodel import col, select -from policyengine_api.data.v2.metadata.parameter_tree_queries import ( +from policyengine_api.data.v2.metadata.parameter_tree_reads import ( parameter_children_from_rows, parameter_children_query, ) -from policyengine_api.data.v2.metadata.query_support import ( - MetadataQueryContext, +from policyengine_api.data.v2.metadata.read_support import ( + MetadataReadContext, MetadataResourceNotFoundError, escape_like, page_result, @@ -60,7 +60,7 @@ def _utc_day_start(selected_time: datetime) -> datetime: ) -class ParameterQueryMethods(MetadataQueryContext): +class ParameterReadMethods(MetadataReadContext): """Read parameters and canonical values from the selected catalog.""" def list_parameters( diff --git a/policyengine_api/data/v2/metadata/parameter_tree_queries.py b/policyengine_api/data/v2/metadata/parameter_tree_reads.py similarity index 98% rename from policyengine_api/data/v2/metadata/parameter_tree_queries.py rename to policyengine_api/data/v2/metadata/parameter_tree_reads.py index d653d000b..238ae92fa 100644 --- a/policyengine_api/data/v2/metadata/parameter_tree_queries.py +++ b/policyengine_api/data/v2/metadata/parameter_tree_reads.py @@ -1,4 +1,4 @@ -"""Database queries for direct parameter-tree children.""" +"""Database reads for direct parameter-tree children.""" from __future__ import annotations diff --git a/policyengine_api/data/v2/metadata/query_support.py b/policyengine_api/data/v2/metadata/read_support.py similarity index 96% rename from policyengine_api/data/v2/metadata/query_support.py rename to policyengine_api/data/v2/metadata/read_support.py index 23d6d3d8d..367cb6a7a 100644 --- a/policyengine_api/data/v2/metadata/query_support.py +++ b/policyengine_api/data/v2/metadata/read_support.py @@ -1,4 +1,4 @@ -"""Shared database execution and pagination for v2 metadata queries.""" +"""Shared database execution and pagination for v2 metadata reads.""" from __future__ import annotations @@ -27,8 +27,8 @@ class InvalidMetadataPageError(ValueError): ResourceT = TypeVar("ResourceT") -class MetadataQueryContext: - """Own the session and catalog selection shared by metadata query methods.""" +class MetadataReadContext: + """Own the session and catalog selection shared by metadata read methods.""" def __init__(self, session: Session, *, running_policyengine_version: str): self._session = session diff --git a/policyengine_api/data/v2/metadata/region_queries.py b/policyengine_api/data/v2/metadata/region_reads.py similarity index 97% rename from policyengine_api/data/v2/metadata/region_queries.py rename to policyengine_api/data/v2/metadata/region_reads.py index 67b5e3df6..6735e45ce 100644 --- a/policyengine_api/data/v2/metadata/region_queries.py +++ b/policyengine_api/data/v2/metadata/region_reads.py @@ -10,8 +10,8 @@ from policyengine_api.data.v2.catalog.catalog_selection import ( MetadataCatalogUnavailableError, ) -from policyengine_api.data.v2.metadata.query_support import ( - MetadataQueryContext, +from policyengine_api.data.v2.metadata.read_support import ( + MetadataReadContext, MetadataResourceNotFoundError, page_result, query_rows, @@ -46,7 +46,7 @@ def _region(region: Region) -> MetadataRegion: ) -class RegionQueryMethods(MetadataQueryContext): +class RegionReadMethods(MetadataReadContext): """Read regions and economy options from the selected catalog.""" def list_regions( diff --git a/policyengine_api/data/v2/metadata/variable_queries.py b/policyengine_api/data/v2/metadata/variable_reads.py similarity index 95% rename from policyengine_api/data/v2/metadata/variable_queries.py rename to policyengine_api/data/v2/metadata/variable_reads.py index 7bbb4bb0d..ff3082a26 100644 --- a/policyengine_api/data/v2/metadata/variable_queries.py +++ b/policyengine_api/data/v2/metadata/variable_reads.py @@ -6,8 +6,8 @@ import sqlalchemy as sa from sqlmodel import col, select -from policyengine_api.data.v2.metadata.query_support import ( - MetadataQueryContext, +from policyengine_api.data.v2.metadata.read_support import ( + MetadataReadContext, MetadataResourceNotFoundError, escape_like, page_result, @@ -36,7 +36,7 @@ def _variable(variable: Variable) -> MetadataVariable: ) -class VariableQueryMethods(MetadataQueryContext): +class VariableReadMethods(MetadataReadContext): """Read variables from the selected catalog.""" def list_variables( diff --git a/policyengine_api/data/v2/policies/__init__.py b/policyengine_api/data/v2/policies/__init__.py index abc6de984..0704ba052 100644 --- a/policyengine_api/data/v2/policies/__init__.py +++ b/policyengine_api/data/v2/policies/__init__.py @@ -1 +1 @@ -"""Immutable v2 policy validation, persistence, and read operations.""" +"""Database create and read operations for immutable v2 policies.""" diff --git a/policyengine_api/data/v2/policies/catalog_resolution.py b/policyengine_api/data/v2/policies/catalog_resolution.py deleted file mode 100644 index c8e37862d..000000000 --- a/policyengine_api/data/v2/policies/catalog_resolution.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Catalog resolution and validation for immutable v2 policy commands.""" - -from __future__ import annotations - -from uuid import UUID - -from sqlmodel import Session, col, select - -from policyengine_api.constants import POLICYENGINE_VERSION -from policyengine_api.data.v2.catalog.catalog_selection import select_catalog -from policyengine_api.data.v2.models import Parameter -from policyengine_api.services.v2.policies.commands import ( - PolicyCreateCommand, - ResolvedPolicyCreateCommand, -) - - -class PolicyCatalogValidationError(ValueError): - """Raised when policy content does not belong to the selected catalog.""" - - -def _version_parameter_ids( - session: Session, - *, - model_version_id: UUID, - requested_ids: set[UUID], -) -> set[UUID]: - if not requested_ids: - return set() - return set( - session.exec( - select(Parameter.id).where( - Parameter.tax_benefit_model_version_id == model_version_id, - col(Parameter.id).in_(requested_ids), - ) - ).all() - ) - - -def resolve_policy_catalog( - session: Session, - command: PolicyCreateCommand, - *, - policyengine_version: str | None = None, - running_policyengine_version: str = POLICYENGINE_VERSION, -) -> ResolvedPolicyCreateCommand: - """Bind validated content to one exact initialized catalog.""" - - selected = select_catalog( - session, - country_id=command.country_id, - running_policyengine_version=running_policyengine_version, - policyengine_version=policyengine_version, - ) - if command.tax_benefit_model_id != selected.model.id: - raise PolicyCatalogValidationError( - "tax_benefit_model_id does not match the selected country catalog" - ) - - requested_parameter_ids = {value.parameter_id for value in command.parameter_values} - resolved_parameter_ids = _version_parameter_ids( - session, - model_version_id=selected.model_version.id, - requested_ids=requested_parameter_ids, - ) - if resolved_parameter_ids != requested_parameter_ids: - raise PolicyCatalogValidationError( - "every parameter_id must belong to the selected model version" - ) - - return ResolvedPolicyCreateCommand( - country_id=command.country_id, - tax_benefit_model_id=selected.model.id, - tax_benefit_model_version_id=selected.model_version.id, - policyengine_version=selected.policyengine_version, - parameter_values=command.parameter_values, - ) diff --git a/policyengine_api/data/v2/policies/creates.py b/policyengine_api/data/v2/policies/creates.py new file mode 100644 index 000000000..204403f11 --- /dev/null +++ b/policyengine_api/data/v2/policies/creates.py @@ -0,0 +1,86 @@ +"""Database creates used by immutable v2 policy operations.""" + +from __future__ import annotations + +from uuid import UUID, uuid4 + +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, col + +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + ParameterValue, + Policy, +) +from policyengine_api.services.v2.policies.commands import ( + ResolvedPolicyCreateCommand, +) + + +def create_policy( + session: Session, + command: ResolvedPolicyCreateCommand, + *, + canonicalization_version: int, + content_hash: str, +) -> UUID | None: + policy_id = uuid4() + statement = ( + insert(Policy) + .values( + id=policy_id, + country_id=command.country_id, + tax_benefit_model_id=command.tax_benefit_model_id, + tax_benefit_model_version_id=command.tax_benefit_model_version_id, + canonicalization_version=canonicalization_version, + content_hash=content_hash, + ) + .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") + .returning(col(Policy.id)) + ) + return session.execute(statement).scalar_one_or_none() + + +def create_parameter_values( + session: Session, + *, + policy_id: UUID, + command: ResolvedPolicyCreateCommand, +) -> None: + session.add_all( + [ + ParameterValue( + policy_id=policy_id, + dynamic_id=None, + parameter_id=value.parameter_id, + value_json=value.value, + start_date=value.start_date, + end_date=value.end_date, + ) + for value in command.parameter_values + ] + ) + session.flush() + + +def create_legacy_policy_mapping( + session: Session, + *, + country_id: str, + legacy_policy_id: int, + source_policy_hash: str, + policy_id: UUID, +) -> UUID | None: + """Create one mapping and return its UUID, or none after a conflict.""" + + return session.execute( + insert(LegacyPolicyMapping) + .values( + country_id=country_id, + legacy_policy_id=legacy_policy_id, + policy_id=policy_id, + source_policy_hash=source_policy_hash, + ) + .on_conflict_do_nothing(constraint="uq_legacy_policy_mappings_country_legacy") + .returning(col(LegacyPolicyMapping.id)) + ).scalar_one_or_none() diff --git a/policyengine_api/data/v2/policies/legacy_mappings.py b/policyengine_api/data/v2/policies/legacy_mappings.py deleted file mode 100644 index 0b5305316..000000000 --- a/policyengine_api/data/v2/policies/legacy_mappings.py +++ /dev/null @@ -1,69 +0,0 @@ -"""SQL operations for durable v1-policy-to-v2-policy mappings.""" - -from __future__ import annotations - -from uuid import UUID - -from sqlalchemy.dialects.postgresql import insert -from sqlmodel import Session, col, select - -from policyengine_api.data.v2.models import LegacyPolicyMapping - - -class LegacyPolicyMappingIntegrityError(RuntimeError): - """Raised when one immutable v1 identity maps inconsistently.""" - - -def find_legacy_policy_mapping( - session: Session, - *, - country_id: str, - legacy_policy_id: int, - lock: bool, -) -> LegacyPolicyMapping | None: - statement = select(LegacyPolicyMapping).where( - LegacyPolicyMapping.country_id == country_id, - LegacyPolicyMapping.legacy_policy_id == legacy_policy_id, - ) - if lock: - statement = statement.with_for_update() - return session.exec(statement).one_or_none() - - -def verify_legacy_policy_mapping( - mapping: LegacyPolicyMapping, - *, - source_policy_hash: str, - expected_policy_id: UUID | None = None, -) -> None: - if mapping.source_policy_hash != source_policy_hash: - raise LegacyPolicyMappingIntegrityError( - "legacy policy identity was presented with a different source hash" - ) - if expected_policy_id is not None and mapping.policy_id != expected_policy_id: - raise LegacyPolicyMappingIntegrityError( - "legacy policy mapping does not match translated immutable content" - ) - - -def insert_legacy_policy_mapping( - session: Session, - *, - country_id: str, - legacy_policy_id: int, - source_policy_hash: str, - policy_id: UUID, -) -> UUID | None: - """Insert one mapping and return its UUID, or none after a conflict.""" - - return session.execute( - insert(LegacyPolicyMapping) - .values( - country_id=country_id, - legacy_policy_id=legacy_policy_id, - policy_id=policy_id, - source_policy_hash=source_policy_hash, - ) - .on_conflict_do_nothing(constraint="uq_legacy_policy_mappings_country_legacy") - .returning(col(LegacyPolicyMapping.id)) - ).scalar_one_or_none() diff --git a/policyengine_api/data/v2/policies/persistence.py b/policyengine_api/data/v2/policies/persistence.py deleted file mode 100644 index 8815f2dfa..000000000 --- a/policyengine_api/data/v2/policies/persistence.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Conflict-aware PostgreSQL persistence for immutable v2 policies.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from uuid import UUID, uuid4 - -from sqlalchemy.dialects.postgresql import insert -from sqlmodel import Session, col, select - -from policyengine_api.data.v2.models import ( - ParameterValue, - Policy, - TaxBenefitModelVersion, -) -from policyengine_api.data.v2.policies.canonicalization import ( - CanonicalPolicyContent, - canonical_policy_document, - canonicalize_policy, -) -from policyengine_api.services.v2.policies.commands import ( - ResolvedPolicyCreateCommand, -) - - -class PolicyPersistenceIntegrityError(RuntimeError): - """Raised when policy hash persistence no longer matches stored content.""" - - -class PolicyContentHashCollisionError(PolicyPersistenceIntegrityError): - """Raised when equal version/hash keys identify different canonical bytes.""" - - -@dataclass(frozen=True) -class PolicyPersistenceResult: - """Inserted or deduplicated immutable policy identity.""" - - policy_id: UUID - created: bool - - -def _insert_policy( - session: Session, - command: ResolvedPolicyCreateCommand, - content: CanonicalPolicyContent, -) -> UUID | None: - policy_id = uuid4() - statement = ( - insert(Policy) - .values( - id=policy_id, - country_id=command.country_id, - tax_benefit_model_id=command.tax_benefit_model_id, - tax_benefit_model_version_id=command.tax_benefit_model_version_id, - canonicalization_version=content.version, - content_hash=content.content_hash, - ) - .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") - .returning(col(Policy.id)) - ) - return session.execute(statement).scalar_one_or_none() - - -def _insert_parameter_values( - session: Session, - *, - policy_id: UUID, - command: ResolvedPolicyCreateCommand, -) -> None: - session.add_all( - [ - ParameterValue( - policy_id=policy_id, - dynamic_id=None, - parameter_id=value.parameter_id, - value_json=value.value, - start_date=value.start_date, - end_date=value.end_date, - ) - for value in command.parameter_values - ] - ) - session.flush() - - -def _stored_policy_command( - session: Session, - policy: Policy, -) -> ResolvedPolicyCreateCommand: - model_version = session.get( - TaxBenefitModelVersion, - policy.tax_benefit_model_version_id, - ) - if model_version is None: - raise PolicyPersistenceIntegrityError( - "stored policy references an absent model version" - ) - values = session.exec( - select(ParameterValue).where(ParameterValue.policy_id == policy.id) - ).all() - return ResolvedPolicyCreateCommand.model_validate( - { - "country_id": policy.country_id, - "tax_benefit_model_id": policy.tax_benefit_model_id, - "tax_benefit_model_version_id": policy.tax_benefit_model_version_id, - "policyengine_version": model_version.version, - "parameter_values": [ - { - "parameter_id": value.parameter_id, - "value": value.value_json, - "start_date": value.start_date, - "end_date": value.end_date, - } - for value in values - ], - } - ) - - -def _existing_policy_after_conflict( - session: Session, - content: CanonicalPolicyContent, -) -> Policy: - policy = session.exec( - select(Policy).where( - Policy.canonicalization_version == content.version, - Policy.content_hash == content.content_hash, - ) - ).one_or_none() - if policy is None: - raise PolicyPersistenceIntegrityError( - "policy hash conflict did not resolve to a stored policy" - ) - return policy - - -def persist_resolved_policy( - session: Session, - command: ResolvedPolicyCreateCommand, - *, - canonicalizer: Callable[ - [ResolvedPolicyCreateCommand], CanonicalPolicyContent - ] = canonicalize_policy, -) -> PolicyPersistenceResult: - """Insert one policy atomically or verify and return equivalent content.""" - - content = canonicalizer(command) - inserted_id = _insert_policy(session, command, content) - if inserted_id is not None: - _insert_parameter_values(session, policy_id=inserted_id, command=command) - return PolicyPersistenceResult(policy_id=inserted_id, created=True) - - existing = _existing_policy_after_conflict(session, content) - stored_document = canonical_policy_document( - _stored_policy_command(session, existing) - ) - if stored_document != content.document: - raise PolicyContentHashCollisionError( - "stored policy content differs for the same canonical version and hash" - ) - return PolicyPersistenceResult(policy_id=existing.id, created=False) diff --git a/policyengine_api/data/v2/policies/queries.py b/policyengine_api/data/v2/policies/reads.py similarity index 50% rename from policyengine_api/data/v2/policies/queries.py rename to policyengine_api/data/v2/policies/reads.py index 6f8c22551..e94f2c5a4 100644 --- a/policyengine_api/data/v2/policies/queries.py +++ b/policyengine_api/data/v2/policies/reads.py @@ -1,4 +1,4 @@ -"""Country-scoped database queries for immutable v2 policies.""" +"""Database reads used by immutable v2 policy operations.""" from __future__ import annotations @@ -9,7 +9,19 @@ from sqlmodel import Session, col, select -from policyengine_api.data.v2.models import Parameter, ParameterValue, Policy +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v2.catalog.catalog_selection import ( + SelectedCatalog, + select_catalog, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.policies.commands import ResolvedPolicyCreateCommand class PolicyNotFoundError(LookupError): @@ -45,6 +57,130 @@ class PolicyPage: has_more: bool +def read_policy_catalog( + session: Session, + country_id: str, + *, + policyengine_version: str | None = None, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> SelectedCatalog: + """Read the exact initialized catalog selected for a policy command.""" + + return select_catalog( + session, + country_id=country_id, + running_policyengine_version=running_policyengine_version, + policyengine_version=policyengine_version, + ) + + +def read_version_parameter_ids( + session: Session, + *, + model_version_id: UUID, + requested_ids: set[UUID], +) -> set[UUID]: + """Read requested parameter IDs that belong to one model version.""" + + if not requested_ids: + return set() + return set( + session.exec( + select(Parameter.id).where( + Parameter.tax_benefit_model_version_id == model_version_id, + col(Parameter.id).in_(requested_ids), + ) + ).all() + ) + + +def read_parameters_by_name( + session: Session, + *, + model_version_id: UUID, + names: set[str], +) -> dict[str, Parameter]: + """Read named parameters that belong to one model version.""" + + if not names: + return {} + parameters = session.exec( + select(Parameter).where( + Parameter.tax_benefit_model_version_id == model_version_id, + col(Parameter.name).in_(names), + ) + ).all() + return {parameter.name: parameter for parameter in parameters} + + +def read_policy_by_content_identity( + session: Session, + *, + canonicalization_version: int, + content_hash: str, +) -> Policy | None: + """Read the policy stored under one canonical version and content hash.""" + + return session.exec( + select(Policy).where( + Policy.canonicalization_version == canonicalization_version, + Policy.content_hash == content_hash, + ) + ).one_or_none() + + +def read_stored_policy_command( + session: Session, + policy: Policy, +) -> ResolvedPolicyCreateCommand | None: + """Read stored policy content in the form used by canonicalization.""" + + model_version = session.get( + TaxBenefitModelVersion, + policy.tax_benefit_model_version_id, + ) + if model_version is None: + return None + values = session.exec( + select(ParameterValue).where(ParameterValue.policy_id == policy.id) + ).all() + return ResolvedPolicyCreateCommand.model_validate( + { + "country_id": policy.country_id, + "tax_benefit_model_id": policy.tax_benefit_model_id, + "tax_benefit_model_version_id": policy.tax_benefit_model_version_id, + "policyengine_version": model_version.version, + "parameter_values": [ + { + "parameter_id": value.parameter_id, + "value": value.value_json, + "start_date": value.start_date, + "end_date": value.end_date, + } + for value in values + ], + } + ) + + +def read_legacy_policy_mapping( + session: Session, + *, + country_id: str, + legacy_policy_id: int, + lock: bool, +) -> LegacyPolicyMapping | None: + """Read one country-scoped legacy policy mapping.""" + + statement = select(LegacyPolicyMapping).where( + LegacyPolicyMapping.country_id == country_id, + LegacyPolicyMapping.legacy_policy_id == legacy_policy_id, + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + def _parameter_values_by_policy( session: Session, policy_ids: list[UUID], diff --git a/policyengine_api/data/v2/user_policies/__init__.py b/policyengine_api/data/v2/user_policies/__init__.py index 2e52406fe..375e2b4dd 100644 --- a/policyengine_api/data/v2/user_policies/__init__.py +++ b/policyengine_api/data/v2/user_policies/__init__.py @@ -1 +1 @@ -"""Native v2 user-policy association operations.""" +"""Database CRUD operations for v2 user-policy associations.""" diff --git a/policyengine_api/data/v2/user_policies/creates.py b/policyengine_api/data/v2/user_policies/creates.py new file mode 100644 index 000000000..c8e7b6904 --- /dev/null +++ b/policyengine_api/data/v2/user_policies/creates.py @@ -0,0 +1,89 @@ +"""Database creates used by v2 user-policy association operations.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, col + +from policyengine_api.data.v2.models import ( + LegacyUserMapping, + LegacyUserPolicyMapping, + User, + UserPolicy, +) +from policyengine_api.services.v2.user_policies.commands import ( + UserPolicyCreateCommand, +) + + +def create_user_policy( + session: Session, + command: UserPolicyCreateCommand, +) -> UserPolicy: + """Create one independently identified association.""" + + association = UserPolicy(**command.model_dump()) + session.add(association) + session.flush() + session.refresh(association) + return association + + +def create_transition_user( + session: Session, + *, + primary_country: str, +) -> User: + """Create a minimal v2 user for one legacy identity.""" + + user = User(primary_country=primary_country) + session.add(user) + session.flush() + return user + + +def create_legacy_user_mapping( + session: Session, + *, + legacy_user_id: str, + user_id: UUID, +) -> UUID | None: + """Create one legacy-user mapping, or return none after a conflict.""" + + return session.execute( + insert(LegacyUserMapping) + .values(legacy_user_id=legacy_user_id, user_id=user_id) + .on_conflict_do_nothing(index_elements=[LegacyUserMapping.legacy_user_id]) + .returning(col(LegacyUserMapping.user_id)) + ).scalar_one_or_none() + + +def create_legacy_user_policy_mapping( + session: Session, + *, + country_id: str, + legacy_user_policy_id: int, + user_policy_id: UUID, + source_revision: int, + fingerprint_version: int, + fingerprint: str, +) -> UUID | None: + """Create one legacy association mapping, or return none after a conflict.""" + + return session.execute( + insert(LegacyUserPolicyMapping) + .values( + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + user_policy_id=user_policy_id, + last_applied_source_revision=source_revision, + fingerprint_version=fingerprint_version, + fingerprint_sha256=fingerprint, + ) + .on_conflict_do_nothing( + constraint="uq_legacy_user_policy_mappings_country_legacy" + ) + .returning(col(LegacyUserPolicyMapping.id)) + ).scalar_one_or_none() diff --git a/policyengine_api/data/v2/user_policies/deletes.py b/policyengine_api/data/v2/user_policies/deletes.py new file mode 100644 index 000000000..70594aeca --- /dev/null +++ b/policyengine_api/data/v2/user_policies/deletes.py @@ -0,0 +1,21 @@ +"""Database deletes used by v2 user-policy association operations.""" + +from __future__ import annotations + +from sqlmodel import Session + +from policyengine_api.data.v2.models import User, UserPolicy + + +def delete_user_policy(session: Session, association: UserPolicy) -> None: + """Delete one association and flush its database cascades.""" + + session.delete(association) + session.flush() + + +def delete_transition_user(session: Session, user: User) -> None: + """Delete an unreferenced transition user after a mapping conflict.""" + + session.delete(user) + session.flush() diff --git a/policyengine_api/data/v2/user_policies/legacy_mappings.py b/policyengine_api/data/v2/user_policies/legacy_mappings.py deleted file mode 100644 index 51c0886fd..000000000 --- a/policyengine_api/data/v2/user_policies/legacy_mappings.py +++ /dev/null @@ -1,278 +0,0 @@ -"""SQL operations for legacy users and saved-policy association mappings.""" - -from __future__ import annotations - -from dataclasses import dataclass -from uuid import UUID - -from sqlalchemy.dialects.postgresql import insert -from sqlmodel import Session, col, select - -from policyengine_api.data.v2.models import ( - LegacyUserMapping, - LegacyUserPolicyMapping, - User, - UserPolicy, -) -from policyengine_api.data.v2.models.base import utc_now -from policyengine_api.services.v2.user_policies.commands import ( - UserPolicyCreateCommand, -) - -USER_POLICY_FINGERPRINT_VERSION = 1 - - -class LegacyUserPolicyIntegrityError(RuntimeError): - """Raised when source, policy, association, or mapping identity conflicts.""" - - -@dataclass(frozen=True) -class LegacyUserPolicyPersistenceResult: - association_id: UUID - policy_id: UUID - association_created: bool - association_updated: bool - mapping_created: bool - - -def _legacy_user_mapping( - session: Session, - legacy_user_id: str, - *, - lock: bool, -) -> LegacyUserMapping | None: - statement = select(LegacyUserMapping).where( - LegacyUserMapping.legacy_user_id == legacy_user_id - ) - if lock: - statement = statement.with_for_update() - return session.exec(statement).one_or_none() - - -def resolve_legacy_user_id( - session: Session, - *, - legacy_user_id: str, - primary_country: str, -) -> UUID: - """Return one durable v2 UUID for an exact opaque v1 user identifier.""" - - existing = _legacy_user_mapping(session, legacy_user_id, lock=True) - if existing is not None: - if session.get(User, existing.user_id) is None: - raise LegacyUserPolicyIntegrityError( - "legacy user mapping has no referenced v2 user" - ) - return existing.user_id - - user = User(primary_country=primary_country) - session.add(user) - session.flush() - inserted_user_id: UUID | None = session.execute( - insert(LegacyUserMapping) - .values( - legacy_user_id=legacy_user_id, - user_id=user.id, - ) - .on_conflict_do_nothing(index_elements=[LegacyUserMapping.legacy_user_id]) - .returning(col(LegacyUserMapping.user_id)) - ).scalar_one_or_none() - if inserted_user_id is not None: - return inserted_user_id - - session.delete(user) - session.flush() - concurrent = _legacy_user_mapping(session, legacy_user_id, lock=False) - if concurrent is None or session.get(User, concurrent.user_id) is None: - raise LegacyUserPolicyIntegrityError( - "legacy user mapping conflict did not resolve to a v2 user" - ) - return concurrent.user_id - - -def _mapping( - session: Session, - *, - country_id: str, - legacy_user_policy_id: int, - lock: bool, -) -> LegacyUserPolicyMapping | None: - statement = select(LegacyUserPolicyMapping).where( - LegacyUserPolicyMapping.country_id == country_id, - LegacyUserPolicyMapping.legacy_user_policy_id == legacy_user_policy_id, - ) - if lock: - statement = statement.with_for_update() - return session.exec(statement).one_or_none() - - -def _mapped_association( - session: Session, - mapping: LegacyUserPolicyMapping, -) -> UserPolicy: - association = session.exec( - select(UserPolicy).where( - UserPolicy.id == mapping.user_policy_id, - UserPolicy.country_id == mapping.country_id, - ) - ).one_or_none() - if association is None: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy mapping has no association" - ) - return association - - -def apply_existing_legacy_user_policy_mapping( - session: Session, - *, - mapping: LegacyUserPolicyMapping, - country_id: str, - reform_label: str | None, - fingerprint: str, - user_id: UUID, - policy_id: UUID, - changed_fields: frozenset[str], - source_revision: int, -) -> LegacyUserPolicyPersistenceResult: - association = _mapped_association(session, mapping) - if ( - association.policy_id != policy_id - or association.country_id != country_id - or association.user_id != user_id - ): - raise LegacyUserPolicyIntegrityError( - "legacy user-policy mapping conflicts with immutable association fields" - ) - if mapping.fingerprint_version != USER_POLICY_FINGERPRINT_VERSION: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy fingerprint version is unsupported" - ) - if source_revision < mapping.last_applied_source_revision: - return LegacyUserPolicyPersistenceResult( - association_id=association.id, - policy_id=policy_id, - association_created=False, - association_updated=False, - mapping_created=False, - ) - if source_revision == mapping.last_applied_source_revision: - if mapping.fingerprint_sha256 != fingerprint: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy revision conflicts with its stored fingerprint" - ) - return LegacyUserPolicyPersistenceResult( - association_id=association.id, - policy_id=policy_id, - association_created=False, - association_updated=False, - mapping_created=False, - ) - if source_revision != mapping.last_applied_source_revision + 1: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy revision has an unapplied predecessor" - ) - - association_updated = ( - "reform_label" in changed_fields and association.name != reform_label - ) - if association_updated: - association.name = reform_label - association.updated_at = utc_now() - session.add(association) - mapping.fingerprint_sha256 = fingerprint - mapping.last_applied_source_revision = source_revision - mapping.updated_at = utc_now() - session.add(mapping) - session.flush() - return LegacyUserPolicyPersistenceResult( - association_id=association.id, - policy_id=policy_id, - association_created=False, - association_updated=association_updated, - mapping_created=False, - ) - - -def persist_legacy_user_policy_mapping( - session: Session, - *, - country_id: str, - legacy_user_policy_id: int, - reform_label: str | None, - projection: UserPolicyCreateCommand, - fingerprint: str, - source_revision: int, - changed_fields: frozenset[str], -) -> LegacyUserPolicyPersistenceResult: - """Create or advance one v1 saved-policy association mapping.""" - - existing = _mapping( - session, - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, - lock=True, - ) - if existing is not None: - return apply_existing_legacy_user_policy_mapping( - session, - mapping=existing, - country_id=country_id, - reform_label=reform_label, - fingerprint=fingerprint, - user_id=projection.user_id, - policy_id=projection.policy_id, - changed_fields=changed_fields, - source_revision=source_revision, - ) - - association = UserPolicy(**projection.model_dump()) - session.add(association) - session.flush() - mapping_id = session.execute( - insert(LegacyUserPolicyMapping) - .values( - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, - user_policy_id=association.id, - last_applied_source_revision=source_revision, - fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, - fingerprint_sha256=fingerprint, - ) - .on_conflict_do_nothing( - constraint="uq_legacy_user_policy_mappings_country_legacy" - ) - .returning(col(LegacyUserPolicyMapping.id)) - ).scalar_one_or_none() - if mapping_id is not None: - return LegacyUserPolicyPersistenceResult( - association_id=association.id, - policy_id=projection.policy_id, - association_created=True, - association_updated=False, - mapping_created=True, - ) - - session.delete(association) - session.flush() - concurrent = _mapping( - session, - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, - lock=False, - ) - if concurrent is None: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy mapping conflict did not resolve to a stored row" - ) - return apply_existing_legacy_user_policy_mapping( - session, - mapping=concurrent, - country_id=country_id, - reform_label=reform_label, - fingerprint=fingerprint, - user_id=projection.user_id, - policy_id=projection.policy_id, - changed_fields=changed_fields, - source_revision=source_revision, - ) diff --git a/policyengine_api/data/v2/user_policies/persistence.py b/policyengine_api/data/v2/user_policies/persistence.py deleted file mode 100644 index 13929bfb2..000000000 --- a/policyengine_api/data/v2/user_policies/persistence.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Transactional database persistence for mutable user-policy associations.""" - -from __future__ import annotations - -from uuid import UUID - -from sqlmodel import Session, select - -from policyengine_api.data.v2.models import Policy, User, UserPolicy -from policyengine_api.data.v2.models.base import utc_now -from policyengine_api.data.v2.user_policies.queries import ( - UserPolicyRead, - association_read, - get_user_policy_row, -) -from policyengine_api.services.v2.user_policies.commands import ( - UserPolicyCreateCommand, - UserPolicyPatchCommand, -) - - -class AssociationPolicyNotFoundError(LookupError): - """Raised when an association references an unknown policy UUID.""" - - -class AssociationUserNotFoundError(LookupError): - """Raised when an association references an unknown v2 user UUID.""" - - -class AssociationCountryConflictError(ValueError): - """Raised when an association and its referenced policy differ by country.""" - - -def create_user_policy( - session: Session, - command: UserPolicyCreateCommand, -) -> UserPolicyRead: - """Create one independently identified association after link validation.""" - - if session.get(User, command.user_id) is None: - raise AssociationUserNotFoundError(f"user {command.user_id} was not found") - - policy = session.exec( - select(Policy).where(Policy.id == command.policy_id) - ).one_or_none() - if policy is None: - raise AssociationPolicyNotFoundError( - f"policy {command.policy_id} was not found" - ) - if policy.country_id != command.country_id: - raise AssociationCountryConflictError( - "Association country_id must match the referenced policy" - ) - association = UserPolicy(**command.model_dump()) - session.add(association) - session.flush() - session.refresh(association) - return association_read(association) - - -def patch_user_policy( - session: Session, - *, - country_id: str, - association_id: UUID, - command: UserPolicyPatchCommand, -) -> UserPolicyRead: - """Change only explicitly supplied presentation fields.""" - - association = get_user_policy_row( - session, - country_id=country_id, - association_id=association_id, - ) - changes = command.model_dump(exclude_unset=True) - for field_name, value in changes.items(): - setattr(association, field_name, value) - association.updated_at = utc_now() - session.add(association) - session.flush() - session.refresh(association) - return association_read(association) - - -def delete_user_policy( - session: Session, - *, - country_id: str, - association_id: UUID, -) -> None: - """Delete one association; database cascades remove only its legacy mapping.""" - - association = get_user_policy_row( - session, - country_id=country_id, - association_id=association_id, - ) - session.delete(association) - session.flush() diff --git a/policyengine_api/data/v2/user_policies/queries.py b/policyengine_api/data/v2/user_policies/reads.py similarity index 58% rename from policyengine_api/data/v2/user_policies/queries.py rename to policyengine_api/data/v2/user_policies/reads.py index b183c1ad7..e6940d77e 100644 --- a/policyengine_api/data/v2/user_policies/queries.py +++ b/policyengine_api/data/v2/user_policies/reads.py @@ -1,4 +1,4 @@ -"""Country-scoped database queries for v2 user-policy associations.""" +"""Database reads used by v2 user-policy association operations.""" from __future__ import annotations @@ -8,7 +8,13 @@ from sqlmodel import Session, col, select -from policyengine_api.data.v2.models import UserPolicy +from policyengine_api.data.v2.models import ( + LegacyUserMapping, + LegacyUserPolicyMapping, + Policy, + User, + UserPolicy, +) class UserPolicyNotFoundError(LookupError): @@ -35,6 +41,66 @@ class UserPolicyPage: has_more: bool +def read_user(session: Session, user_id: UUID) -> User | None: + """Read one v2 user by UUID.""" + + return session.get(User, user_id) + + +def read_policy_for_association(session: Session, policy_id: UUID) -> Policy | None: + """Read one policy referenced by an association create command.""" + + return session.exec(select(Policy).where(Policy.id == policy_id)).one_or_none() + + +def read_legacy_user_mapping( + session: Session, + legacy_user_id: str, + *, + lock: bool, +) -> LegacyUserMapping | None: + """Read one legacy-user-to-v2-user mapping.""" + + statement = select(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == legacy_user_id + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def read_legacy_user_policy_mapping( + session: Session, + *, + country_id: str, + legacy_user_policy_id: int, + lock: bool, +) -> LegacyUserPolicyMapping | None: + """Read one country-scoped legacy association mapping.""" + + statement = select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.country_id == country_id, + LegacyUserPolicyMapping.legacy_user_policy_id == legacy_user_policy_id, + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def read_mapped_user_policy( + session: Session, + mapping: LegacyUserPolicyMapping, +) -> UserPolicy | None: + """Read the association referenced by one legacy mapping.""" + + return session.exec( + select(UserPolicy).where( + UserPolicy.id == mapping.user_policy_id, + UserPolicy.country_id == mapping.country_id, + ) + ).one_or_none() + + def association_read(association: UserPolicy) -> UserPolicyRead: return UserPolicyRead( id=association.id, diff --git a/policyengine_api/data/v2/user_policies/updates.py b/policyengine_api/data/v2/user_policies/updates.py new file mode 100644 index 000000000..393bc730d --- /dev/null +++ b/policyengine_api/data/v2/user_policies/updates.py @@ -0,0 +1,48 @@ +"""Database updates used by v2 user-policy association operations.""" + +from __future__ import annotations + +from sqlmodel import Session + +from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy +from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.services.v2.user_policies.commands import UserPolicyPatchCommand + + +def update_user_policy( + session: Session, + association: UserPolicy, + command: UserPolicyPatchCommand, +) -> UserPolicy: + """Update only explicitly supplied association presentation fields.""" + + for field_name, value in command.model_dump(exclude_unset=True).items(): + setattr(association, field_name, value) + association.updated_at = utc_now() + session.add(association) + session.flush() + session.refresh(association) + return association + + +def update_legacy_user_policy_state( + session: Session, + *, + association: UserPolicy, + mapping: LegacyUserPolicyMapping, + reform_label: str | None, + update_name: bool, + fingerprint: str, + source_revision: int, +) -> None: + """Update an association projection and its applied source revision.""" + + if update_name: + association.name = reform_label + association.updated_at = utc_now() + session.add(association) + mapping.fingerprint_sha256 = fingerprint + mapping.last_applied_source_revision = source_revision + mapping.updated_at = utc_now() + session.add(mapping) + session.flush() diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 1a0028f9b..632595117 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -27,8 +27,8 @@ MetadataRegion, MetadataVariable, ) - from policyengine_api.data.v2.policies.queries import PolicyPage, PolicyRead - from policyengine_api.data.v2.user_policies.queries import ( + from policyengine_api.data.v2.policies.reads import PolicyPage, PolicyRead + from policyengine_api.data.v2.user_policies.reads import ( UserPolicyPage, UserPolicyRead, ) diff --git a/policyengine_api/fastapi_routes/v2/policies/response_models.py b/policyengine_api/fastapi_routes/v2/policies/response_models.py index b75db5883..f1549dc73 100644 --- a/policyengine_api/fastapi_routes/v2/policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/policies/response_models.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue, StringConstraints -from policyengine_api.data.v2.policies.queries import PolicyPage, PolicyRead +from policyengine_api.data.v2.policies.reads import PolicyPage, PolicyRead class StrictPolicyAPIModel(BaseModel): diff --git a/policyengine_api/fastapi_routes/v2/policies/routes.py b/policyengine_api/fastapi_routes/v2/policies/routes.py index a9195a536..7260f12fb 100644 --- a/policyengine_api/fastapi_routes/v2/policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/policies/routes.py @@ -27,14 +27,14 @@ PolicyPageResponse, PolicyPageResult, ) -from policyengine_api.data.v2.policies.catalog_resolution import ( +from policyengine_api.data.v2.policies.reads import PolicyNotFoundError +from policyengine_api.services.v2.policies.catalog_validation import ( PolicyCatalogValidationError, ) -from policyengine_api.data.v2.policies.persistence import ( +from policyengine_api.services.v2.policies.creation import ( PolicyContentHashCollisionError, - PolicyPersistenceIntegrityError, + PolicyCreationIntegrityError, ) -from policyengine_api.data.v2.policies.queries import PolicyNotFoundError from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import ( NativeRouteDependencies, @@ -104,7 +104,7 @@ def _policy_operation( return policy_error_response(404, str(error)) except PolicyContentHashCollisionError: return policy_error_response(409, "Policy content hash conflicts with storage") - except PolicyPersistenceIntegrityError: + except PolicyCreationIntegrityError: return policy_error_response(500, "Stored policy integrity failed") except (V2ConfigurationError, MetadataCatalogUnavailableError, SQLAlchemyError): return policy_error_response(503, "V2 policy persistence is unavailable") diff --git a/policyengine_api/fastapi_routes/v2/user_policies/response_models.py b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py index e82db0dd9..e01f167a9 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, StringConstraints -from policyengine_api.data.v2.user_policies.queries import ( +from policyengine_api.data.v2.user_policies.reads import ( UserPolicyPage, UserPolicyRead, ) diff --git a/policyengine_api/fastapi_routes/v2/user_policies/routes.py b/policyengine_api/fastapi_routes/v2/user_policies/routes.py index bf13a726d..6bdd84524 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/routes.py @@ -24,10 +24,10 @@ UserPolicyPageResponse, UserPolicyPageResult, ) -from policyengine_api.data.v2.user_policies.queries import ( +from policyengine_api.data.v2.user_policies.reads import ( UserPolicyNotFoundError, ) -from policyengine_api.data.v2.user_policies.persistence import ( +from policyengine_api.services.v2.user_policies.service import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py index 151d85d78..1c37f2d2e 100644 --- a/policyengine_api/services/policy_mirroring.py +++ b/policyengine_api/services/policy_mirroring.py @@ -12,15 +12,15 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.catalog_resolution import ( +from policyengine_api.services.v2.policies.catalog_validation import ( PolicyCatalogValidationError, ) -from policyengine_api.data.v2.policies.legacy_mappings import ( - LegacyPolicyMappingIntegrityError, -) -from policyengine_api.data.v2.policies.persistence import ( +from policyengine_api.services.v2.policies.creation import ( PolicyContentHashCollisionError, - PolicyPersistenceIntegrityError, + PolicyCreationIntegrityError, +) +from policyengine_api.services.v2.policies.legacy_service import ( + LegacyPolicyMappingIntegrityError, ) from policyengine_api.services.v2.policies.legacy_service import ( LegacyPolicyPersistenceResult, @@ -71,7 +71,7 @@ def _failure_category(error: Exception) -> str: ( LegacyPolicyMappingIntegrityError, PolicyContentHashCollisionError, - PolicyPersistenceIntegrityError, + PolicyCreationIntegrityError, ), ): return "integrity" diff --git a/policyengine_api/services/user_policy_mirroring.py b/policyengine_api/services/user_policy_mirroring.py index 0e0efd1fa..5d7ce4430 100644 --- a/policyengine_api/services/user_policy_mirroring.py +++ b/policyengine_api/services/user_policy_mirroring.py @@ -12,17 +12,17 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.catalog_resolution import ( +from policyengine_api.services.v2.policies.catalog_validation import ( PolicyCatalogValidationError, ) -from policyengine_api.data.v2.policies.legacy_mappings import ( - LegacyPolicyMappingIntegrityError, -) -from policyengine_api.data.v2.policies.persistence import ( +from policyengine_api.services.v2.policies.creation import ( PolicyContentHashCollisionError, - PolicyPersistenceIntegrityError, + PolicyCreationIntegrityError, +) +from policyengine_api.services.v2.policies.legacy_service import ( + LegacyPolicyMappingIntegrityError, ) -from policyengine_api.data.v2.user_policies.legacy_mappings import ( +from policyengine_api.services.v2.user_policies.legacy_service import ( LegacyUserPolicyIntegrityError, LegacyUserPolicyPersistenceResult, ) @@ -85,7 +85,7 @@ def _failure_category(error: Exception) -> str: LegacyPolicyMappingIntegrityError, LegacyUserPolicyIntegrityError, PolicyContentHashCollisionError, - PolicyPersistenceIntegrityError, + PolicyCreationIntegrityError, ), ): return "integrity" diff --git a/policyengine_api/services/user_policy_service.py b/policyengine_api/services/user_policy_service.py index 5f5782884..15035978c 100644 --- a/policyengine_api/services/user_policy_service.py +++ b/policyengine_api/services/user_policy_service.py @@ -17,7 +17,7 @@ from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent -from policyengine_api.data.v2.user_policies.legacy_mappings import ( +from policyengine_api.services.v2.user_policies.legacy_service import ( LegacyUserPolicyPersistenceResult, ) from policyengine_api.services.v2.user_policies.legacy_translation import ( diff --git a/policyengine_api/services/v2/metadata/service.py b/policyengine_api/services/v2/metadata/service.py index 88420c098..333f87db3 100644 --- a/policyengine_api/services/v2/metadata/service.py +++ b/policyengine_api/services/v2/metadata/service.py @@ -9,25 +9,25 @@ UnsupportedPreviewCountryError, validate_policyengine_version, ) -from policyengine_api.data.v2.metadata.dataset_queries import ( - DatasetQueryMethods, +from policyengine_api.data.v2.metadata.dataset_reads import ( + DatasetReadMethods, ) -from policyengine_api.data.v2.metadata.model_queries import ( - ModelQueryMethods, +from policyengine_api.data.v2.metadata.model_reads import ( + ModelReadMethods, ) -from policyengine_api.data.v2.metadata.parameter_queries import ( - ParameterQueryMethods, +from policyengine_api.data.v2.metadata.parameter_reads import ( + ParameterReadMethods, ) -from policyengine_api.data.v2.metadata.query_support import ( +from policyengine_api.data.v2.metadata.read_support import ( InvalidMetadataPageError, MetadataResourceNotFoundError, validate_metadata_page, ) -from policyengine_api.data.v2.metadata.region_queries import ( - RegionQueryMethods, +from policyengine_api.data.v2.metadata.region_reads import ( + RegionReadMethods, ) -from policyengine_api.data.v2.metadata.variable_queries import ( - VariableQueryMethods, +from policyengine_api.data.v2.metadata.variable_reads import ( + VariableReadMethods, ) @@ -45,10 +45,10 @@ class V2MetadataService( - ModelQueryMethods, - VariableQueryMethods, - ParameterQueryMethods, - DatasetQueryMethods, - RegionQueryMethods, + ModelReadMethods, + VariableReadMethods, + ParameterReadMethods, + DatasetReadMethods, + RegionReadMethods, ): - """Expose resource-specific metadata query methods to the route layer.""" + """Expose resource-specific metadata read methods to the route layer.""" diff --git a/policyengine_api/data/v2/policies/canonicalization.py b/policyengine_api/services/v2/policies/canonicalization.py similarity index 97% rename from policyengine_api/data/v2/policies/canonicalization.py rename to policyengine_api/services/v2/policies/canonicalization.py index 524a5f2ec..5e45a568b 100644 --- a/policyengine_api/data/v2/policies/canonicalization.py +++ b/policyengine_api/services/v2/policies/canonicalization.py @@ -1,4 +1,4 @@ -"""Versioned canonical content identity for immutable v2 policies.""" +"""Database-independent content identity for immutable v2 policies.""" from __future__ import annotations diff --git a/policyengine_api/services/v2/policies/catalog_validation.py b/policyengine_api/services/v2/policies/catalog_validation.py new file mode 100644 index 000000000..bdb0c12ba --- /dev/null +++ b/policyengine_api/services/v2/policies/catalog_validation.py @@ -0,0 +1,43 @@ +"""Database-independent catalog validation for immutable v2 policies.""" + +from __future__ import annotations + +from uuid import UUID + +from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog +from policyengine_api.services.v2.policies.commands import ( + PolicyCreateCommand, + ResolvedPolicyCreateCommand, +) + + +class PolicyCatalogValidationError(ValueError): + """Raised when policy content does not belong to the selected catalog.""" + + +def validate_policy_catalog( + command: PolicyCreateCommand, + *, + selected: SelectedCatalog, + resolved_parameter_ids: set[UUID], +) -> ResolvedPolicyCreateCommand: + """Validate preloaded catalog records and bind them to policy content.""" + + if command.tax_benefit_model_id != selected.model.id: + raise PolicyCatalogValidationError( + "tax_benefit_model_id does not match the selected country catalog" + ) + + requested_parameter_ids = {value.parameter_id for value in command.parameter_values} + if resolved_parameter_ids != requested_parameter_ids: + raise PolicyCatalogValidationError( + "every parameter_id must belong to the selected model version" + ) + + return ResolvedPolicyCreateCommand( + country_id=command.country_id, + tax_benefit_model_id=selected.model.id, + tax_benefit_model_version_id=selected.model_version.id, + policyengine_version=selected.policyengine_version, + parameter_values=command.parameter_values, + ) diff --git a/policyengine_api/services/v2/policies/creation.py b/policyengine_api/services/v2/policies/creation.py new file mode 100644 index 000000000..17dbf3815 --- /dev/null +++ b/policyengine_api/services/v2/policies/creation.py @@ -0,0 +1,123 @@ +"""Application sequencing for validated immutable policy creation.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from uuid import UUID + +from sqlmodel import Session + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v2.policies.creates import ( + create_parameter_values, + create_policy, +) +from policyengine_api.data.v2.policies.reads import ( + read_policy_by_content_identity, + read_policy_catalog, + read_stored_policy_command, + read_version_parameter_ids, +) +from policyengine_api.services.v2.policies.canonicalization import ( + CanonicalPolicyContent, + canonical_policy_document, + canonicalize_policy, +) +from policyengine_api.services.v2.policies.catalog_validation import ( + validate_policy_catalog, +) +from policyengine_api.services.v2.policies.commands import ( + PolicyCreateCommand, + ResolvedPolicyCreateCommand, +) + + +class PolicyCreationIntegrityError(RuntimeError): + """Raised when stored policy content cannot support safe deduplication.""" + + +class PolicyContentHashCollisionError(PolicyCreationIntegrityError): + """Raised when equal version/hash keys identify different canonical bytes.""" + + +@dataclass(frozen=True) +class PolicyCreationResult: + """New or deduplicated immutable policy identity.""" + + policy_id: UUID + created: bool + + +def resolve_policy_catalog( + session: Session, + command: PolicyCreateCommand, + *, + policyengine_version: str | None = None, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> ResolvedPolicyCreateCommand: + """Read and validate the exact catalog selected for policy creation.""" + + selected = read_policy_catalog( + session, + command.country_id, + policyengine_version=policyengine_version, + running_policyengine_version=running_policyengine_version, + ) + requested_parameter_ids = {value.parameter_id for value in command.parameter_values} + resolved_parameter_ids = read_version_parameter_ids( + session, + model_version_id=selected.model_version.id, + requested_ids=requested_parameter_ids, + ) + return validate_policy_catalog( + command, + selected=selected, + resolved_parameter_ids=resolved_parameter_ids, + ) + + +def create_resolved_policy( + session: Session, + command: ResolvedPolicyCreateCommand, + *, + canonicalizer: Callable[ + [ResolvedPolicyCreateCommand], CanonicalPolicyContent + ] = canonicalize_policy, +) -> PolicyCreationResult: + """Create one policy or verify and return equivalent stored content.""" + + content = canonicalizer(command) + created_policy_id = create_policy( + session, + command, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if created_policy_id is not None: + create_parameter_values( + session, + policy_id=created_policy_id, + command=command, + ) + return PolicyCreationResult(policy_id=created_policy_id, created=True) + + existing = read_policy_by_content_identity( + session, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if existing is None: + raise PolicyCreationIntegrityError( + "policy hash conflict did not resolve to a stored policy" + ) + stored_command = read_stored_policy_command(session, existing) + if stored_command is None: + raise PolicyCreationIntegrityError( + "stored policy references an absent model version" + ) + if canonical_policy_document(stored_command) != content.document: + raise PolicyContentHashCollisionError( + "stored policy content differs for the same canonical version and hash" + ) + return PolicyCreationResult(policy_id=existing.id, created=False) diff --git a/policyengine_api/services/v2/policies/legacy_service.py b/policyengine_api/services/v2/policies/legacy_service.py index 633b648f7..4d013aee5 100644 --- a/policyengine_api/services/v2/policies/legacy_service.py +++ b/policyengine_api/services/v2/policies/legacy_service.py @@ -9,14 +9,15 @@ from sqlmodel import Session from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION -from policyengine_api.data.v2.policies.legacy_mappings import ( - LegacyPolicyMappingIntegrityError, - find_legacy_policy_mapping, - insert_legacy_policy_mapping, - verify_legacy_policy_mapping, +from policyengine_api.data.v2.models import LegacyPolicyMapping +from policyengine_api.data.v2.policies.creates import ( + create_legacy_policy_mapping, ) -from policyengine_api.data.v2.policies.persistence import ( - persist_resolved_policy, +from policyengine_api.data.v2.policies.reads import ( + read_legacy_policy_mapping, +) +from policyengine_api.services.v2.policies.creation import ( + create_resolved_policy, ) from policyengine_api.services.v2.policies.legacy_translation import ( LegacyPolicySnapshot, @@ -33,6 +34,28 @@ class LegacyPolicyPersistenceResult: mapping_created: bool +class LegacyPolicyMappingIntegrityError(RuntimeError): + """Raised when one immutable v1 identity maps inconsistently.""" + + +def verify_legacy_policy_mapping( + mapping: LegacyPolicyMapping, + *, + source_policy_hash: str, + expected_policy_id: UUID | None = None, +) -> None: + """Validate an existing mapping without performing SQL.""" + + if mapping.source_policy_hash != source_policy_hash: + raise LegacyPolicyMappingIntegrityError( + "legacy policy identity was presented with a different source hash" + ) + if expected_policy_id is not None and mapping.policy_id != expected_policy_id: + raise LegacyPolicyMappingIntegrityError( + "legacy policy mapping does not match translated immutable content" + ) + + def persist_legacy_policy( session: Session, snapshot: LegacyPolicySnapshot, @@ -42,7 +65,7 @@ def persist_legacy_policy( ) -> LegacyPolicyPersistenceResult: """Translate, deduplicate, and map one v1 policy in the caller transaction.""" - existing = find_legacy_policy_mapping( + existing = read_legacy_policy_mapping( session, country_id=snapshot.country_id, legacy_policy_id=snapshot.legacy_policy_id, @@ -60,7 +83,7 @@ def persist_legacy_policy( running_policyengine_version=running_policyengine_version, country_package_versions=country_package_versions, ) - policy_result = persist_resolved_policy(session, command) + policy_result = create_resolved_policy(session, command) if existing is not None: verify_legacy_policy_mapping( existing, @@ -73,7 +96,7 @@ def persist_legacy_policy( mapping_created=False, ) - mapping_id = insert_legacy_policy_mapping( + mapping_id = create_legacy_policy_mapping( session, country_id=snapshot.country_id, legacy_policy_id=snapshot.legacy_policy_id, @@ -87,7 +110,7 @@ def persist_legacy_policy( mapping_created=True, ) - concurrent = find_legacy_policy_mapping( + concurrent = read_legacy_policy_mapping( session, country_id=snapshot.country_id, legacy_policy_id=snapshot.legacy_policy_id, diff --git a/policyengine_api/services/v2/policies/legacy_translation.py b/policyengine_api/services/v2/policies/legacy_translation.py index 190be22eb..a597b1e96 100644 --- a/policyengine_api/services/v2/policies/legacy_translation.py +++ b/policyengine_api/services/v2/policies/legacy_translation.py @@ -5,19 +5,17 @@ from collections.abc import Mapping from datetime import date, datetime, time, timezone from typing import Annotated -from uuid import UUID from policyengine_core.periods import ( # type: ignore[import-untyped] period as parse_policyengine_period, ) from pydantic import Field, field_validator -from sqlmodel import Session, col, select +from sqlmodel import Session from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION -from policyengine_api.data.v2.catalog.catalog_selection import select_catalog -from policyengine_api.data.v2.models import Parameter -from policyengine_api.data.v2.policies.catalog_resolution import ( - resolve_policy_catalog, +from policyengine_api.data.v2.policies.reads import ( + read_parameters_by_name, + read_policy_catalog, ) from policyengine_api.query_parameters import CountryId from policyengine_api.services.v2.policies.commands import ( @@ -27,6 +25,9 @@ StrictJsonValue, StrictPolicyCommand, ) +from policyengine_api.services.v2.policies.catalog_validation import ( + validate_policy_catalog, +) class LegacyPolicyTranslationError(ValueError): @@ -89,23 +90,6 @@ def parse_legacy_period(value: str) -> tuple[datetime, datetime]: return start_date, end_date -def _parameters_by_name( - session: Session, - *, - model_version_id: UUID, - names: set[str], -) -> dict[str, Parameter]: - if not names: - return {} - parameters = session.exec( - select(Parameter).where( - Parameter.tax_benefit_model_version_id == model_version_id, - col(Parameter.name).in_(names), - ) - ).all() - return {parameter.name: parameter for parameter in parameters} - - def translate_legacy_policy( session: Session, snapshot: LegacyPolicySnapshot, @@ -120,15 +104,15 @@ def translate_legacy_policy( raise LegacyPolicyTranslationError( "legacy policy api_version does not match the running country package" ) - selected = select_catalog( + selected = read_policy_catalog( session, - country_id=snapshot.country_id, + snapshot.country_id, running_policyengine_version=running_policyengine_version, ) policy_json = snapshot.policy_json assert isinstance(policy_json, dict) parameter_names = set(policy_json) - parameters = _parameters_by_name( + parameters = read_parameters_by_name( session, model_version_id=selected.model_version.id, names=parameter_names, @@ -170,8 +154,8 @@ def translate_legacy_policy( raise LegacyPolicyTranslationError( "legacy parameter periods or values conflict" ) from error - return resolve_policy_catalog( - session, + return validate_policy_catalog( command, - running_policyengine_version=running_policyengine_version, + selected=selected, + resolved_parameter_ids={parameter.id for parameter in parameters.values()}, ) diff --git a/policyengine_api/services/v2/policies/service.py b/policyengine_api/services/v2/policies/service.py index 1dddd2d64..eee7ce6d7 100644 --- a/policyengine_api/services/v2/policies/service.py +++ b/policyengine_api/services/v2/policies/service.py @@ -9,9 +9,7 @@ from sqlmodel import Session from policyengine_api.constants import POLICYENGINE_VERSION -from policyengine_api.data.v2.policies.catalog_resolution import resolve_policy_catalog -from policyengine_api.data.v2.policies.persistence import persist_resolved_policy -from policyengine_api.data.v2.policies.queries import ( +from policyengine_api.data.v2.policies.reads import ( PolicyPage, PolicyRead, list_policies, @@ -21,6 +19,10 @@ NativePolicyCreateCommand, PolicyCreateCommand, ) +from policyengine_api.services.v2.policies.creation import ( + create_resolved_policy, + resolve_policy_catalog, +) from policyengine_api.services.v2.policies.legacy_service import ( LegacyPolicyPersistenceResult, persist_legacy_policy, @@ -64,7 +66,7 @@ def create_policy( policyengine_version=command.policyengine_version, running_policyengine_version=self._running_policyengine_version, ) - persisted = persist_resolved_policy(session, resolved) + persisted = create_resolved_policy(session, resolved) item = read_policy( session, country_id=command.country_id, diff --git a/policyengine_api/services/v2/user_policies/legacy_service.py b/policyengine_api/services/v2/user_policies/legacy_service.py index 9fe8673be..0eb26e6e9 100644 --- a/policyengine_api/services/v2/user_policies/legacy_service.py +++ b/policyengine_api/services/v2/user_policies/legacy_service.py @@ -2,13 +2,30 @@ from __future__ import annotations +from dataclasses import dataclass +from uuid import UUID + from sqlmodel import Session -from policyengine_api.data.v2.user_policies.legacy_mappings import ( - LegacyUserPolicyIntegrityError, - LegacyUserPolicyPersistenceResult, - persist_legacy_user_policy_mapping, - resolve_legacy_user_id, +from policyengine_api.data.v2.models import LegacyUserPolicyMapping +from policyengine_api.data.v2.user_policies.creates import ( + create_legacy_user_mapping, + create_legacy_user_policy_mapping, + create_transition_user, + create_user_policy, +) +from policyengine_api.data.v2.user_policies.deletes import ( + delete_transition_user, + delete_user_policy, +) +from policyengine_api.data.v2.user_policies.reads import ( + read_legacy_user_mapping, + read_legacy_user_policy_mapping, + read_mapped_user_policy, + read_user, +) +from policyengine_api.data.v2.user_policies.updates import ( + update_legacy_user_policy_state, ) from policyengine_api.services.v2.policies.legacy_service import ( persist_legacy_policy, @@ -16,13 +33,216 @@ from policyengine_api.services.v2.policies.legacy_translation import ( LegacyPolicySnapshot, ) +from policyengine_api.services.v2.user_policies.commands import ( + UserPolicyCreateCommand, +) from policyengine_api.services.v2.user_policies.legacy_translation import ( + USER_POLICY_FINGERPRINT_VERSION, LegacyUserPolicySnapshot, fingerprint_legacy_user_policy, project_legacy_user_policy, ) +class LegacyUserPolicyIntegrityError(RuntimeError): + """Raised when source, policy, association, or mapping identity conflicts.""" + + +@dataclass(frozen=True) +class LegacyUserPolicyPersistenceResult: + association_id: UUID + policy_id: UUID + association_created: bool + association_updated: bool + mapping_created: bool + + +def resolve_legacy_user_id( + session: Session, + *, + legacy_user_id: str, + primary_country: str, +) -> UUID: + """Return one durable v2 UUID for an exact opaque v1 user identifier.""" + + existing = read_legacy_user_mapping(session, legacy_user_id, lock=True) + if existing is not None: + if read_user(session, existing.user_id) is None: + raise LegacyUserPolicyIntegrityError( + "legacy user mapping has no referenced v2 user" + ) + return existing.user_id + + user = create_transition_user(session, primary_country=primary_country) + created_user_id = create_legacy_user_mapping( + session, + legacy_user_id=legacy_user_id, + user_id=user.id, + ) + if created_user_id is not None: + return created_user_id + + delete_transition_user(session, user) + concurrent = read_legacy_user_mapping(session, legacy_user_id, lock=False) + if concurrent is None or read_user(session, concurrent.user_id) is None: + raise LegacyUserPolicyIntegrityError( + "legacy user mapping conflict did not resolve to a v2 user" + ) + return concurrent.user_id + + +def apply_existing_legacy_user_policy_mapping( + session: Session, + *, + mapping: LegacyUserPolicyMapping, + country_id: str, + reform_label: str | None, + fingerprint: str, + user_id: UUID, + policy_id: UUID, + changed_fields: frozenset[str], + source_revision: int, +) -> LegacyUserPolicyPersistenceResult: + """Validate and, when newer, update one existing association mapping.""" + + association = read_mapped_user_policy(session, mapping) + if association is None: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping has no association" + ) + if ( + association.policy_id != policy_id + or association.country_id != country_id + or association.user_id != user_id + ): + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping conflicts with immutable association fields" + ) + if mapping.fingerprint_version != USER_POLICY_FINGERPRINT_VERSION: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy fingerprint version is unsupported" + ) + if source_revision < mapping.last_applied_source_revision: + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=False, + mapping_created=False, + ) + if source_revision == mapping.last_applied_source_revision: + if mapping.fingerprint_sha256 != fingerprint: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy revision conflicts with its stored fingerprint" + ) + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=False, + mapping_created=False, + ) + if source_revision != mapping.last_applied_source_revision + 1: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy revision has an unapplied predecessor" + ) + + association_updated = ( + "reform_label" in changed_fields and association.name != reform_label + ) + update_legacy_user_policy_state( + session, + association=association, + mapping=mapping, + reform_label=reform_label, + update_name=association_updated, + fingerprint=fingerprint, + source_revision=source_revision, + ) + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=association_updated, + mapping_created=False, + ) + + +def persist_legacy_user_policy_mapping( + session: Session, + *, + country_id: str, + legacy_user_policy_id: int, + reform_label: str | None, + projection: UserPolicyCreateCommand, + fingerprint: str, + source_revision: int, + changed_fields: frozenset[str], +) -> LegacyUserPolicyPersistenceResult: + """Create or advance one v1 saved-policy association mapping.""" + + existing = read_legacy_user_policy_mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + lock=True, + ) + if existing is not None: + return apply_existing_legacy_user_policy_mapping( + session, + mapping=existing, + country_id=country_id, + reform_label=reform_label, + fingerprint=fingerprint, + user_id=projection.user_id, + policy_id=projection.policy_id, + changed_fields=changed_fields, + source_revision=source_revision, + ) + + association = create_user_policy(session, projection) + mapping_id = create_legacy_user_policy_mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + user_policy_id=association.id, + source_revision=source_revision, + fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, + fingerprint=fingerprint, + ) + if mapping_id is not None: + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=projection.policy_id, + association_created=True, + association_updated=False, + mapping_created=True, + ) + + delete_user_policy(session, association) + concurrent = read_legacy_user_policy_mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + lock=False, + ) + if concurrent is None: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping conflict did not resolve to a stored row" + ) + return apply_existing_legacy_user_policy_mapping( + session, + mapping=concurrent, + country_id=country_id, + reform_label=reform_label, + fingerprint=fingerprint, + user_id=projection.user_id, + policy_id=projection.policy_id, + changed_fields=changed_fields, + source_revision=source_revision, + ) + + def persist_legacy_user_policy( session: Session, snapshot: LegacyUserPolicySnapshot, diff --git a/policyengine_api/services/v2/user_policies/legacy_translation.py b/policyengine_api/services/v2/user_policies/legacy_translation.py index eb994e812..f4352b08e 100644 --- a/policyengine_api/services/v2/user_policies/legacy_translation.py +++ b/policyengine_api/services/v2/user_policies/legacy_translation.py @@ -9,9 +9,6 @@ from pydantic import Field -from policyengine_api.data.v2.user_policies.legacy_mappings import ( - USER_POLICY_FINGERPRINT_VERSION, -) from policyengine_api.query_parameters import CountryId, LegacyUserId from policyengine_api.services.v2.policies.commands import StrictPolicyCommand from policyengine_api.services.v2.user_policies.commands import ( @@ -19,6 +16,9 @@ ) +USER_POLICY_FINGERPRINT_VERSION = 1 + + class LegacyUserPolicySnapshot(StrictPolicyCommand): """Detached complete committed v1 saved-policy row.""" diff --git a/policyengine_api/services/v2/user_policies/service.py b/policyengine_api/services/v2/user_policies/service.py index b67caee55..1487df6b5 100644 --- a/policyengine_api/services/v2/user_policies/service.py +++ b/policyengine_api/services/v2/user_policies/service.py @@ -7,16 +7,24 @@ from sqlalchemy.orm import sessionmaker from sqlmodel import Session -from policyengine_api.data.v2.user_policies.queries import ( +from policyengine_api.data.v2.user_policies.creates import ( + create_user_policy as create_user_policy_row, +) +from policyengine_api.data.v2.user_policies.deletes import ( + delete_user_policy as delete_user_policy_row, +) +from policyengine_api.data.v2.user_policies.reads import ( UserPolicyPage, UserPolicyRead, + association_read, + get_user_policy_row, list_user_policies, + read_policy_for_association, read_user_policy, + read_user, ) -from policyengine_api.data.v2.user_policies.persistence import ( - create_user_policy, - delete_user_policy, - patch_user_policy, +from policyengine_api.data.v2.user_policies.updates import ( + update_user_policy, ) from policyengine_api.services.v2.policies.legacy_translation import ( LegacyPolicySnapshot, @@ -34,6 +42,18 @@ ) +class AssociationPolicyNotFoundError(LookupError): + """Raised when an association references an unknown policy UUID.""" + + +class AssociationUserNotFoundError(LookupError): + """Raised when an association references an unknown v2 user UUID.""" + + +class AssociationCountryConflictError(ValueError): + """Raised when an association and its referenced policy differ by country.""" + + class V2UserPolicyService: """Own transaction boundaries for native association operations.""" @@ -45,7 +65,20 @@ def create_user_policy( command: UserPolicyCreateCommand, ) -> UserPolicyRead: with self._sessions.begin() as session: - return create_user_policy(session, command) + if read_user(session, command.user_id) is None: + raise AssociationUserNotFoundError( + f"user {command.user_id} was not found" + ) + policy = read_policy_for_association(session, command.policy_id) + if policy is None: + raise AssociationPolicyNotFoundError( + f"policy {command.policy_id} was not found" + ) + if policy.country_id != command.country_id: + raise AssociationCountryConflictError( + "Association country_id must match the referenced policy" + ) + return association_read(create_user_policy_row(session, command)) def get_user_policy( self, @@ -87,12 +120,12 @@ def patch_user_policy( command: UserPolicyPatchCommand, ) -> UserPolicyRead: with self._sessions.begin() as session: - return patch_user_policy( + association = get_user_policy_row( session, country_id=country_id, association_id=association_id, - command=command, ) + return association_read(update_user_policy(session, association, command)) def delete_user_policy( self, @@ -101,11 +134,12 @@ def delete_user_policy( association_id: UUID, ) -> None: with self._sessions.begin() as session: - delete_user_policy( + association = get_user_policy_row( session, country_id=country_id, association_id=association_id, ) + delete_user_policy_row(session, association) def mirror_legacy_user_policy( self, diff --git a/tests/integration/test_v2_policy_persistence.py b/tests/integration/test_v2_policy_persistence.py index 81705c3f8..680f72626 100644 --- a/tests/integration/test_v2_policy_persistence.py +++ b/tests/integration/test_v2_policy_persistence.py @@ -24,17 +24,17 @@ TaxBenefitModel, TaxBenefitModelVersion, ) -from policyengine_api.data.v2.policies.canonicalization import ( +from policyengine_api.services.v2.policies.canonicalization import ( CanonicalPolicyContent, canonical_policy_document, canonicalize_policy, ) -from policyengine_api.data.v2.policies.legacy_mappings import ( +from policyengine_api.services.v2.policies.legacy_service import ( LegacyPolicyMappingIntegrityError, ) -from policyengine_api.data.v2.policies.persistence import ( +from policyengine_api.services.v2.policies.creation import ( PolicyContentHashCollisionError, - persist_resolved_policy, + create_resolved_policy, ) from policyengine_api.services.v2.policies.commands import ( ResolvedPolicyCreateCommand, @@ -152,10 +152,10 @@ def test_equivalent_create_returns_one_policy_and_one_child_set() -> None: model, version, parameter = _catalog(session) model_id = model.id command = _command(model.id, version.id, parameter.id) - first = persist_resolved_policy(session, command) + first = create_resolved_policy(session, command) with Session(engine) as session, session.begin(): - second = persist_resolved_policy(session, command) + second = create_resolved_policy(session, command) assert first.created is True assert second.created is False @@ -190,7 +190,7 @@ def test_equal_hash_with_different_canonical_bytes_is_an_integrity_error() -> No parameter_id = parameter.id original = _command(model_id, version_id, parameter_id, value=0.2) stored = canonicalize_policy(original) - persist_resolved_policy(session, original) + create_resolved_policy(session, original) changed = _command(model_id, version_id, parameter_id, value=0.3) @@ -205,7 +205,7 @@ def simulated_collision( with Session(engine) as session, session.begin(): with pytest.raises(PolicyContentHashCollisionError): - persist_resolved_policy( + create_resolved_policy( session, changed, canonicalizer=simulated_collision, @@ -364,7 +364,7 @@ def test_concurrent_equivalent_creates_return_one_policy_uuid() -> None: def create(): with Session(engine) as session, session.begin(): barrier.wait() - return persist_resolved_policy(session, command) + return create_resolved_policy(session, command) with ThreadPoolExecutor(max_workers=2) as executor: futures = [executor.submit(create) for _ in range(2)] @@ -404,12 +404,12 @@ def test_empty_and_distinct_policy_content_persist_independently() -> None: policyengine_version=version.version, parameter_values=[], ) - first = persist_resolved_policy(session, empty) - second = persist_resolved_policy( + first = create_resolved_policy(session, empty) + second = create_resolved_policy( session, _command(model.id, version.id, parameter.id, value=1), ) - third = persist_resolved_policy( + third = create_resolved_policy( session, _command(model.id, version.id, parameter.id, value=2), ) @@ -440,7 +440,7 @@ def test_child_insert_failure_rolls_back_the_policy_and_all_values() -> None: with pytest.raises(IntegrityError): with Session(engine) as session, session.begin(): - persist_resolved_policy(session, invalid) + create_resolved_policy(session, invalid) with Session(engine) as session: policy_count = session.scalar( diff --git a/tests/integration/test_v2_user_policy_mirroring.py b/tests/integration/test_v2_user_policy_mirroring.py index 791e2d60e..4ed540360 100644 --- a/tests/integration/test_v2_user_policy_mirroring.py +++ b/tests/integration/test_v2_user_policy_mirroring.py @@ -29,7 +29,7 @@ User, UserPolicy, ) -from policyengine_api.data.v2.user_policies.legacy_mappings import ( +from policyengine_api.services.v2.user_policies.legacy_service import ( resolve_legacy_user_id, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL diff --git a/tests/unit/services/test_policy_mirroring.py b/tests/unit/services/test_policy_mirroring.py index e41378832..2232fdb6c 100644 --- a/tests/unit/services/test_policy_mirroring.py +++ b/tests/unit/services/test_policy_mirroring.py @@ -8,7 +8,7 @@ import pytest from sqlalchemy.exc import OperationalError, TimeoutError -from policyengine_api.data.v2.policies.legacy_mappings import ( +from policyengine_api.services.v2.policies.legacy_service import ( LegacyPolicyMappingIntegrityError, ) from policyengine_api.services.v2.policies.legacy_service import ( diff --git a/tests/unit/services/test_user_policy_mirroring.py b/tests/unit/services/test_user_policy_mirroring.py index 3fb0d4195..8f19ad6d1 100644 --- a/tests/unit/services/test_user_policy_mirroring.py +++ b/tests/unit/services/test_user_policy_mirroring.py @@ -10,7 +10,7 @@ from sqlalchemy.exc import OperationalError, TimeoutError from policyengine_api.data.v1_models import UserPolicyMirrorEvent -from policyengine_api.data.v2.user_policies.legacy_mappings import ( +from policyengine_api.services.v2.user_policies.legacy_service import ( LegacyUserPolicyIntegrityError, LegacyUserPolicyPersistenceResult, ) diff --git a/tests/unit/services/test_user_policy_service.py b/tests/unit/services/test_user_policy_service.py index aae90f3cf..e6aaf9f0f 100644 --- a/tests/unit/services/test_user_policy_service.py +++ b/tests/unit/services/test_user_policy_service.py @@ -14,7 +14,7 @@ ) from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent -from policyengine_api.data.v2.user_policies.legacy_mappings import ( +from policyengine_api.services.v2.user_policies.legacy_service import ( LegacyUserPolicyPersistenceResult, ) from policyengine_api.services.user_policy_service import ( diff --git a/tests/unit/v2/test_data_crud_boundaries.py b/tests/unit/v2/test_data_crud_boundaries.py new file mode 100644 index 000000000..e7e2c2c4e --- /dev/null +++ b/tests/unit/v2/test_data_crud_boundaries.py @@ -0,0 +1,89 @@ +"""Structural checks for API v2 database CRUD modules.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + + +PROJECT_ROOT = Path(__file__).parents[3] +DATA_ROOT = PROJECT_ROOT / "policyengine_api" / "data" / "v2" +SERVICE_ROOT = PROJECT_ROOT / "policyengine_api" / "services" / "v2" / "policies" + + +def _tree(path: Path) -> ast.Module: + return ast.parse(path.read_text(encoding="utf-8")) + + +def _called_names(path: Path) -> set[str]: + names: set[str] = set() + for node in ast.walk(_tree(path)): + if not isinstance(node, ast.Call): + continue + if isinstance(node.func, ast.Name): + names.add(node.func.id) + elif isinstance(node.func, ast.Attribute): + names.add(node.func.attr) + return names + + +def _imported_modules(path: Path) -> set[str]: + modules: set[str] = set() + for node in ast.walk(_tree(path)): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + modules.add(node.module or "") + return modules + + +@pytest.mark.parametrize( + "relative_path", + ( + "policies/creates.py", + "user_policies/creates.py", + "user_policies/updates.py", + "user_policies/deletes.py", + ), +) +def test_mutation_modules_contain_no_database_reads(relative_path: str) -> None: + calls = _called_names(DATA_ROOT / relative_path) + assert "select" not in calls + assert "get" not in calls + + +@pytest.mark.parametrize( + "relative_path", + ( + "policies/reads.py", + "user_policies/reads.py", + "metadata/dataset_reads.py", + "metadata/model_reads.py", + "metadata/parameter_reads.py", + "metadata/parameter_tree_reads.py", + "metadata/region_reads.py", + "metadata/variable_reads.py", + ), +) +def test_read_modules_contain_no_database_mutations(relative_path: str) -> None: + calls = _called_names(DATA_ROOT / relative_path) + assert calls.isdisjoint({"insert", "add", "add_all", "delete"}) + + +@pytest.mark.parametrize( + "filename", + ("catalog_validation.py", "canonicalization.py"), +) +def test_policy_validation_and_identity_modules_have_no_database_dependency( + filename: str, +) -> None: + imports = _imported_modules(SERVICE_ROOT / filename) + assert not any( + module == "sqlalchemy" + or module.startswith("sqlalchemy.") + or module == "sqlmodel" + or module.startswith("sqlmodel.") + for module in imports + ) diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index d24372120..a8df1f3ac 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -551,17 +551,17 @@ def test_economy_options_require_a_national_region_and_dataset( _service(catalog_session).get_economy_options("us") -def test_query_modules_import_no_policyengine_or_v1_metadata_source() -> None: +def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: data_directory = Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" modules = ( data_directory / "catalog" / "catalog_selection.py", - data_directory / "metadata" / "dataset_queries.py", - data_directory / "metadata" / "model_queries.py", - data_directory / "metadata" / "parameter_queries.py", - data_directory / "metadata" / "parameter_tree_queries.py", - data_directory / "metadata" / "query_support.py", - data_directory / "metadata" / "region_queries.py", - data_directory / "metadata" / "variable_queries.py", + data_directory / "metadata" / "dataset_reads.py", + data_directory / "metadata" / "model_reads.py", + data_directory / "metadata" / "parameter_reads.py", + data_directory / "metadata" / "parameter_tree_reads.py", + data_directory / "metadata" / "read_support.py", + data_directory / "metadata" / "region_reads.py", + data_directory / "metadata" / "variable_reads.py", ) imported = set() for module in modules: @@ -593,26 +593,26 @@ def test_query_modules_import_no_policyengine_or_v1_metadata_source() -> None: ) -def test_resource_service_methods_are_defined_in_their_query_modules() -> None: +def test_resource_service_methods_are_defined_in_their_read_modules() -> None: expected_modules = { - "list_models": "model_queries", - "get_model": "model_queries", - "get_model_by_country": "model_queries", - "list_model_versions": "model_queries", - "get_model_version": "model_queries", - "list_variables": "variable_queries", - "get_variable": "variable_queries", - "list_parameters": "parameter_queries", - "get_parameter": "parameter_queries", - "list_parameter_children": "parameter_queries", - "list_parameter_values": "parameter_queries", - "get_parameter_value": "parameter_queries", - "list_datasets": "dataset_queries", - "get_dataset": "dataset_queries", - "list_regions": "region_queries", - "get_region": "region_queries", - "get_region_by_code": "region_queries", - "get_economy_options": "region_queries", + "list_models": "model_reads", + "get_model": "model_reads", + "get_model_by_country": "model_reads", + "list_model_versions": "model_reads", + "get_model_version": "model_reads", + "list_variables": "variable_reads", + "get_variable": "variable_reads", + "list_parameters": "parameter_reads", + "get_parameter": "parameter_reads", + "list_parameter_children": "parameter_reads", + "list_parameter_values": "parameter_reads", + "get_parameter_value": "parameter_reads", + "list_datasets": "dataset_reads", + "get_dataset": "dataset_reads", + "list_regions": "region_reads", + "get_region": "region_reads", + "get_region_by_code": "region_reads", + "get_economy_options": "region_reads", } for method_name, module_name in expected_modules.items(): diff --git a/tests/unit/v2/test_policy_canonicalization.py b/tests/unit/v2/test_policy_canonicalization.py index 4a361a9b0..bf92fb9da 100644 --- a/tests/unit/v2/test_policy_canonicalization.py +++ b/tests/unit/v2/test_policy_canonicalization.py @@ -6,7 +6,7 @@ import hashlib from uuid import UUID, uuid4 -from policyengine_api.data.v2.policies.canonicalization import ( +from policyengine_api.services.v2.policies.canonicalization import ( POLICY_CANONICALIZATION_VERSION, canonical_policy_document, canonicalize_policy, diff --git a/tests/unit/v2/test_policy_catalog.py b/tests/unit/v2/test_policy_catalog.py index 9d61b83ed..c09a30b10 100644 --- a/tests/unit/v2/test_policy_catalog.py +++ b/tests/unit/v2/test_policy_catalog.py @@ -16,8 +16,10 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.data.v2.policies.catalog_resolution import ( +from policyengine_api.services.v2.policies.catalog_validation import ( PolicyCatalogValidationError, +) +from policyengine_api.services.v2.policies.creation import ( resolve_policy_catalog, ) from policyengine_api.services.v2.policies.commands import PolicyCreateCommand diff --git a/tests/unit/v2/test_policy_persistence_statements.py b/tests/unit/v2/test_policy_persistence_statements.py index f51716d86..b9ac908ec 100644 --- a/tests/unit/v2/test_policy_persistence_statements.py +++ b/tests/unit/v2/test_policy_persistence_statements.py @@ -4,11 +4,11 @@ from sqlalchemy.dialects import postgresql -from policyengine_api.data.v2.policies import persistence +from policyengine_api.data.v2.policies import creates def test_policy_insert_uses_the_content_identity_constraint_and_returning() -> None: - source = persistence._insert_policy.__code__.co_consts + source = creates.create_policy.__code__.co_consts statement_text = " ".join(str(value) for value in source) assert "uq_policies_canonicalization_content_hash" in statement_text @@ -16,9 +16,9 @@ def test_policy_insert_uses_the_content_identity_constraint_and_returning() -> N # Compile a representative statement through the same PostgreSQL dialect # construct to prove this module does not use a read-before-write insert. statement = ( - persistence.insert(persistence.Policy) + creates.insert(creates.Policy) .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") - .returning(persistence.Policy.id) + .returning(creates.Policy.id) ) compiled = str(statement.compile(dialect=postgresql.dialect())) assert "ON CONFLICT ON CONSTRAINT" in compiled diff --git a/tests/unit/v2/test_policy_query.py b/tests/unit/v2/test_policy_query.py index ed5fa15ab..96fe15fe4 100644 --- a/tests/unit/v2/test_policy_query.py +++ b/tests/unit/v2/test_policy_query.py @@ -16,7 +16,7 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.data.v2.policies.queries import ( +from policyengine_api.data.v2.policies.reads import ( PolicyNotFoundError, list_policies, read_policy, diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py index 7890e2e5f..254674420 100644 --- a/tests/unit/v2/test_policy_routes.py +++ b/tests/unit/v2/test_policy_routes.py @@ -15,18 +15,18 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.catalog_resolution import ( - PolicyCatalogValidationError, -) -from policyengine_api.data.v2.policies.queries import ( +from policyengine_api.data.v2.policies.reads import ( PolicyNotFoundError, PolicyPage, PolicyParameterValueRead, PolicyRead, ) -from policyengine_api.data.v2.policies.persistence import ( +from policyengine_api.services.v2.policies.catalog_validation import ( + PolicyCatalogValidationError, +) +from policyengine_api.services.v2.policies.creation import ( PolicyContentHashCollisionError, - PolicyPersistenceIntegrityError, + PolicyCreationIntegrityError, ) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -307,7 +307,7 @@ def test_list_passes_exact_model_filter_and_canonical_pagination() -> None: "conflicts", ), ( - PolicyPersistenceIntegrityError("database statement secret"), + PolicyCreationIntegrityError("database statement secret"), 500, "integrity", ), diff --git a/tests/unit/v2/test_user_policy_legacy.py b/tests/unit/v2/test_user_policy_legacy.py index bcf78d079..95e606432 100644 --- a/tests/unit/v2/test_user_policy_legacy.py +++ b/tests/unit/v2/test_user_policy_legacy.py @@ -8,7 +8,7 @@ import pytest from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy -from policyengine_api.data.v2.user_policies.legacy_mappings import ( +from policyengine_api.services.v2.user_policies.legacy_service import ( LegacyUserPolicyIntegrityError, apply_existing_legacy_user_policy_mapping, ) diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py index f24720adb..f9eea38d5 100644 --- a/tests/unit/v2/test_user_policy_routes.py +++ b/tests/unit/v2/test_user_policy_routes.py @@ -12,12 +12,12 @@ from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.data.v2.settings import V2ConfigurationError -from policyengine_api.data.v2.user_policies.queries import ( +from policyengine_api.data.v2.user_policies.reads import ( UserPolicyNotFoundError, UserPolicyPage, UserPolicyRead, ) -from policyengine_api.data.v2.user_policies.persistence import ( +from policyengine_api.services.v2.user_policies.service import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py index 40b432bb5..7653a110a 100644 --- a/tests/unit/v2/test_user_policy_service.py +++ b/tests/unit/v2/test_user_policy_service.py @@ -21,10 +21,10 @@ UserPolicy, V2_METADATA, ) -from policyengine_api.data.v2.user_policies.queries import ( +from policyengine_api.data.v2.user_policies.reads import ( UserPolicyNotFoundError, ) -from policyengine_api.data.v2.user_policies.persistence import ( +from policyengine_api.services.v2.user_policies.service import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, From 1847f7b82a12942747e7afb92cef60cb2547a1d2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:36:36 +0400 Subject: [PATCH 10/18] Organize API v2 metadata access by CRUD action --- .../skills/v2-code-organization.md | 17 ++- policyengine_api/data/v2/metadata/__init__.py | 2 +- .../data/v2/metadata/read_support.py | 112 ------------------ .../v2/metadata/{read_models.py => reads.py} | 106 ++++++++++++++++- .../{dataset_reads.py => reads_datasets.py} | 4 +- .../{model_reads.py => reads_models.py} | 4 +- ..._tree_reads.py => reads_parameter_tree.py} | 2 +- ...parameter_reads.py => reads_parameters.py} | 6 +- .../{region_reads.py => reads_regions.py} | 4 +- .../{variable_reads.py => reads_variables.py} | 4 +- .../fastapi_routes/dependencies.py | 2 +- .../v2/metadata/geography_routes.py | 2 +- .../v2/metadata/response_models.py | 2 +- .../services/v2/metadata/service.py | 12 +- tests/unit/v2/test_data_crud_boundaries.py | 13 +- tests/unit/v2/test_metadata_routes.py | 2 +- tests/unit/v2/test_metadata_service.py | 50 ++++---- 17 files changed, 171 insertions(+), 173 deletions(-) delete mode 100644 policyengine_api/data/v2/metadata/read_support.py rename policyengine_api/data/v2/metadata/{read_models.py => reads.py} (50%) rename policyengine_api/data/v2/metadata/{dataset_reads.py => reads_datasets.py} (95%) rename policyengine_api/data/v2/metadata/{model_reads.py => reads_models.py} (96%) rename policyengine_api/data/v2/metadata/{parameter_tree_reads.py => reads_parameter_tree.py} (98%) rename policyengine_api/data/v2/metadata/{parameter_reads.py => reads_parameters.py} (97%) rename policyengine_api/data/v2/metadata/{region_reads.py => reads_regions.py} (98%) rename policyengine_api/data/v2/metadata/{variable_reads.py => reads_variables.py} (96%) diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index 273ca54bb..ab08236de 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -72,9 +72,13 @@ user_policies/ updates.py deletes.py metadata/ - read_models.py - read_support.py - *_reads.py + reads.py + reads_datasets.py + reads_models.py + reads_parameter_tree.py + reads_parameters.py + reads_regions.py + reads_variables.py ``` Database-access modules are organized by SQL operation rather than by HTTP @@ -87,8 +91,11 @@ those calls and owns the transaction. Do not create an empty CRUD module for an operation the resource does not support. Immutable policies therefore have only `creates.py` and `reads.py`. -Mutable user-policy associations have all four modules. Read-only metadata -uses resource-specific `*_reads.py` modules and shared `read_support.py`. +Mutable user-policy associations have all four modules. Read-only metadata uses +shared read behavior and result types in `reads.py`. When that module would +become too broad, append a resource descriptor after the operation name, as in +`reads_variables.py`. Apply the same operation-first naming to future metadata +creates, updates, or deletes, and do not add modules for unsupported operations. Legacy mapping SQL follows the same division: mapping selection belongs in `reads.py`, mapping insertion in `creates.py`, mapping mutation in `updates.py`, diff --git a/policyengine_api/data/v2/metadata/__init__.py b/policyengine_api/data/v2/metadata/__init__.py index 16af473a3..6d505f62f 100644 --- a/policyengine_api/data/v2/metadata/__init__.py +++ b/policyengine_api/data/v2/metadata/__init__.py @@ -1 +1 @@ -"""Read models and database reads for API v2 metadata.""" +"""CRUD-organized database access for API v2 metadata.""" diff --git a/policyengine_api/data/v2/metadata/read_support.py b/policyengine_api/data/v2/metadata/read_support.py deleted file mode 100644 index 367cb6a7a..000000000 --- a/policyengine_api/data/v2/metadata/read_support.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Shared database execution and pagination for v2 metadata reads.""" - -from __future__ import annotations - -from typing import Any, TypeVar - -from sqlalchemy.exc import SQLAlchemyError -from sqlmodel import Session - -from policyengine_api.data.v2.catalog.catalog_selection import ( - MetadataCatalogUnavailableError, - SelectedCatalog, - select_catalog as select_metadata_catalog, - validate_policyengine_version, -) -from policyengine_api.data.v2.metadata.read_models import MetadataPageResult - - -class MetadataResourceNotFoundError(LookupError): - """Raised when a selected catalog does not contain a requested resource.""" - - -class InvalidMetadataPageError(ValueError): - """Raised when collection pagination is outside the documented bounds.""" - - -ResourceT = TypeVar("ResourceT") - - -class MetadataReadContext: - """Own the session and catalog selection shared by metadata read methods.""" - - def __init__(self, session: Session, *, running_policyengine_version: str): - self._session = session - self._running_policyengine_version = validate_policyengine_version( - running_policyengine_version - ) - - def close(self) -> None: - """Close the request-owned read session.""" - - self._session.close() - - def select_catalog( - self, - country_id: str, - policyengine_version: str | None = None, - ) -> SelectedCatalog: - """Select exactly one initialized country catalog.""" - - return select_metadata_catalog( - self._session, - country_id=country_id, - running_policyengine_version=self._running_policyengine_version, - policyengine_version=policyengine_version, - ) - - def _select_paginated_catalog( - self, - country_id: str, - policyengine_version: str | None, - *, - offset: int, - limit: int, - ) -> SelectedCatalog: - validate_metadata_page(offset, limit) - return self.select_catalog(country_id, policyengine_version) - - -def page_result( - selected: SelectedCatalog, - rows: list[ResourceT], - *, - offset: int, - limit: int, -) -> MetadataPageResult[ResourceT]: - """Return one bounded response page from a limit-plus-one query.""" - - return MetadataPageResult( - policyengine_version=selected.policyengine_version, - items=rows[:limit], - offset=offset, - limit=limit, - has_more=len(rows) > limit, - ) - - -def validate_metadata_page(offset: int, limit: int) -> tuple[int, int]: - """Validate the shared v2 metadata collection bounds.""" - - if offset < 0: - raise InvalidMetadataPageError("offset must be at least 0") - if not 1 <= limit <= 500: - raise InvalidMetadataPageError("limit must be between 1 and 500") - return offset, limit - - -def query_rows(session: Session, statement: Any) -> list[Any]: - """Execute one read statement and translate database failures.""" - - try: - return list(session.exec(statement).all()) - except SQLAlchemyError as error: - raise MetadataCatalogUnavailableError( - "the v2 metadata catalog cannot be queried" - ) from error - - -def escape_like(value: str) -> str: - """Escape SQL LIKE wildcard characters in a literal search value.""" - - return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") diff --git a/policyengine_api/data/v2/metadata/read_models.py b/policyengine_api/data/v2/metadata/reads.py similarity index 50% rename from policyengine_api/data/v2/metadata/read_models.py rename to policyengine_api/data/v2/metadata/reads.py index 273319215..55beb40ee 100644 --- a/policyengine_api/data/v2/metadata/read_models.py +++ b/policyengine_api/data/v2/metadata/reads.py @@ -1,13 +1,22 @@ -"""Framework-neutral read models for API v2 metadata resources.""" +"""Shared database reads and typed read results for API v2 metadata.""" from __future__ import annotations from datetime import datetime from enum import StrEnum -from typing import Generic, Literal, TypeVar +from typing import Any, Generic, Literal, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, JsonValue +from sqlalchemy.exc import SQLAlchemyError +from sqlmodel import Session + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + SelectedCatalog, + select_catalog as select_metadata_catalog, + validate_policyengine_version, +) class StrictResponseModel(BaseModel): @@ -150,3 +159,96 @@ class MetadataEconomyOptionsResult(StrictResponseModel): region: list[MetadataRegionOption] time_period: list[MetadataTimePeriodOption] datasets: list[MetadataDatasetOption] + + +class MetadataResourceNotFoundError(LookupError): + """Raised when a selected catalog does not contain a requested resource.""" + + +class InvalidMetadataPageError(ValueError): + """Raised when collection pagination is outside the documented bounds.""" + + +class MetadataReadContext: + """Own the session and catalog selection shared by metadata read methods.""" + + def __init__(self, session: Session, *, running_policyengine_version: str): + self._session = session + self._running_policyengine_version = validate_policyengine_version( + running_policyengine_version + ) + + def close(self) -> None: + """Close the request-owned read session.""" + + self._session.close() + + def select_catalog( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> SelectedCatalog: + """Select exactly one initialized country catalog.""" + + return select_metadata_catalog( + self._session, + country_id=country_id, + running_policyengine_version=self._running_policyengine_version, + policyengine_version=policyengine_version, + ) + + def _select_paginated_catalog( + self, + country_id: str, + policyengine_version: str | None, + *, + offset: int, + limit: int, + ) -> SelectedCatalog: + validate_metadata_page(offset, limit) + return self.select_catalog(country_id, policyengine_version) + + +def page_result( + selected: SelectedCatalog, + rows: list[ResourceT], + *, + offset: int, + limit: int, +) -> MetadataPageResult[ResourceT]: + """Return one bounded response page from a limit-plus-one query.""" + + return MetadataPageResult( + policyengine_version=selected.policyengine_version, + items=rows[:limit], + offset=offset, + limit=limit, + has_more=len(rows) > limit, + ) + + +def validate_metadata_page(offset: int, limit: int) -> tuple[int, int]: + """Validate the shared v2 metadata collection bounds.""" + + if offset < 0: + raise InvalidMetadataPageError("offset must be at least 0") + if not 1 <= limit <= 500: + raise InvalidMetadataPageError("limit must be between 1 and 500") + return offset, limit + + +def query_rows(session: Session, statement: Any) -> list[Any]: + """Execute one read statement and translate database failures.""" + + try: + return list(session.exec(statement).all()) + except SQLAlchemyError as error: + raise MetadataCatalogUnavailableError( + "the v2 metadata catalog cannot be queried" + ) from error + + +def escape_like(value: str) -> str: + """Escape SQL LIKE wildcard characters in a literal search value.""" + + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") diff --git a/policyengine_api/data/v2/metadata/dataset_reads.py b/policyengine_api/data/v2/metadata/reads_datasets.py similarity index 95% rename from policyengine_api/data/v2/metadata/dataset_reads.py rename to policyengine_api/data/v2/metadata/reads_datasets.py index b55fab08c..080a97fa5 100644 --- a/policyengine_api/data/v2/metadata/dataset_reads.py +++ b/policyengine_api/data/v2/metadata/reads_datasets.py @@ -5,13 +5,13 @@ from uuid import UUID from sqlmodel import col, select -from policyengine_api.data.v2.metadata.read_support import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataReadContext, MetadataResourceNotFoundError, page_result, query_rows, ) -from policyengine_api.data.v2.metadata.read_models import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataDataset, MetadataDetailResult, MetadataPageResult, diff --git a/policyengine_api/data/v2/metadata/model_reads.py b/policyengine_api/data/v2/metadata/reads_models.py similarity index 96% rename from policyengine_api/data/v2/metadata/model_reads.py rename to policyengine_api/data/v2/metadata/reads_models.py index 85622b986..0eb568a37 100644 --- a/policyengine_api/data/v2/metadata/model_reads.py +++ b/policyengine_api/data/v2/metadata/reads_models.py @@ -5,12 +5,12 @@ from uuid import UUID from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog -from policyengine_api.data.v2.metadata.read_support import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataReadContext, MetadataResourceNotFoundError, page_result, ) -from policyengine_api.data.v2.metadata.read_models import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataDetailResult, MetadataModel, MetadataModelSelectionResult, diff --git a/policyengine_api/data/v2/metadata/parameter_tree_reads.py b/policyengine_api/data/v2/metadata/reads_parameter_tree.py similarity index 98% rename from policyengine_api/data/v2/metadata/parameter_tree_reads.py rename to policyengine_api/data/v2/metadata/reads_parameter_tree.py index 238ae92fa..fa469f2b3 100644 --- a/policyengine_api/data/v2/metadata/parameter_tree_reads.py +++ b/policyengine_api/data/v2/metadata/reads_parameter_tree.py @@ -8,7 +8,7 @@ import sqlalchemy as sa from sqlmodel import col -from policyengine_api.data.v2.metadata.read_models import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataParameterChild, MetadataParameterSummary, ) diff --git a/policyengine_api/data/v2/metadata/parameter_reads.py b/policyengine_api/data/v2/metadata/reads_parameters.py similarity index 97% rename from policyengine_api/data/v2/metadata/parameter_reads.py rename to policyengine_api/data/v2/metadata/reads_parameters.py index 9be07114e..130514a20 100644 --- a/policyengine_api/data/v2/metadata/parameter_reads.py +++ b/policyengine_api/data/v2/metadata/reads_parameters.py @@ -7,18 +7,18 @@ import sqlalchemy as sa from sqlmodel import col, select -from policyengine_api.data.v2.metadata.parameter_tree_reads import ( +from policyengine_api.data.v2.metadata.reads_parameter_tree import ( parameter_children_from_rows, parameter_children_query, ) -from policyengine_api.data.v2.metadata.read_support import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataReadContext, MetadataResourceNotFoundError, escape_like, page_result, query_rows, ) -from policyengine_api.data.v2.metadata.read_models import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataCanonicalParameterValue, MetadataDetailResult, MetadataPageResult, diff --git a/policyengine_api/data/v2/metadata/region_reads.py b/policyengine_api/data/v2/metadata/reads_regions.py similarity index 98% rename from policyengine_api/data/v2/metadata/region_reads.py rename to policyengine_api/data/v2/metadata/reads_regions.py index 6735e45ce..603476479 100644 --- a/policyengine_api/data/v2/metadata/region_reads.py +++ b/policyengine_api/data/v2/metadata/reads_regions.py @@ -10,13 +10,13 @@ from policyengine_api.data.v2.catalog.catalog_selection import ( MetadataCatalogUnavailableError, ) -from policyengine_api.data.v2.metadata.read_support import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataReadContext, MetadataResourceNotFoundError, page_result, query_rows, ) -from policyengine_api.data.v2.metadata.read_models import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataDatasetOption, MetadataDetailResult, MetadataEconomyOptionsResult, diff --git a/policyengine_api/data/v2/metadata/variable_reads.py b/policyengine_api/data/v2/metadata/reads_variables.py similarity index 96% rename from policyengine_api/data/v2/metadata/variable_reads.py rename to policyengine_api/data/v2/metadata/reads_variables.py index ff3082a26..b7e1ff97e 100644 --- a/policyengine_api/data/v2/metadata/variable_reads.py +++ b/policyengine_api/data/v2/metadata/reads_variables.py @@ -6,14 +6,14 @@ import sqlalchemy as sa from sqlmodel import col, select -from policyengine_api.data.v2.metadata.read_support import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataReadContext, MetadataResourceNotFoundError, escape_like, page_result, query_rows, ) -from policyengine_api.data.v2.metadata.read_models import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataDetailResult, MetadataPageResult, MetadataVariable, diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 632595117..b20ab0554 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -13,7 +13,7 @@ from policyengine_api.json_types import JSONObject if TYPE_CHECKING: - from policyengine_api.data.v2.metadata.read_models import ( + from policyengine_api.data.v2.metadata.reads import ( MetadataCanonicalParameterValue, MetadataDataset, MetadataDetailResult, diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index e9a7705b6..5cd541ab3 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Query from starlette.responses import JSONResponse -from policyengine_api.data.v2.metadata.read_models import MetadataRegionType +from policyengine_api.data.v2.metadata.reads import MetadataRegionType from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataDatasetDetailResponse, MetadataDatasetPageResponse, diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index 9d0d00a2e..433d7426b 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -6,7 +6,7 @@ from pydantic import StringConstraints -from policyengine_api.data.v2.metadata.read_models import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataCanonicalParameterValue, MetadataDataset, MetadataDetailResult, diff --git a/policyengine_api/services/v2/metadata/service.py b/policyengine_api/services/v2/metadata/service.py index 333f87db3..1c743f4a4 100644 --- a/policyengine_api/services/v2/metadata/service.py +++ b/policyengine_api/services/v2/metadata/service.py @@ -9,24 +9,24 @@ UnsupportedPreviewCountryError, validate_policyengine_version, ) -from policyengine_api.data.v2.metadata.dataset_reads import ( +from policyengine_api.data.v2.metadata.reads_datasets import ( DatasetReadMethods, ) -from policyengine_api.data.v2.metadata.model_reads import ( +from policyengine_api.data.v2.metadata.reads_models import ( ModelReadMethods, ) -from policyengine_api.data.v2.metadata.parameter_reads import ( +from policyengine_api.data.v2.metadata.reads_parameters import ( ParameterReadMethods, ) -from policyengine_api.data.v2.metadata.read_support import ( +from policyengine_api.data.v2.metadata.reads import ( InvalidMetadataPageError, MetadataResourceNotFoundError, validate_metadata_page, ) -from policyengine_api.data.v2.metadata.region_reads import ( +from policyengine_api.data.v2.metadata.reads_regions import ( RegionReadMethods, ) -from policyengine_api.data.v2.metadata.variable_reads import ( +from policyengine_api.data.v2.metadata.reads_variables import ( VariableReadMethods, ) diff --git a/tests/unit/v2/test_data_crud_boundaries.py b/tests/unit/v2/test_data_crud_boundaries.py index e7e2c2c4e..70cdc42fe 100644 --- a/tests/unit/v2/test_data_crud_boundaries.py +++ b/tests/unit/v2/test_data_crud_boundaries.py @@ -59,12 +59,13 @@ def test_mutation_modules_contain_no_database_reads(relative_path: str) -> None: ( "policies/reads.py", "user_policies/reads.py", - "metadata/dataset_reads.py", - "metadata/model_reads.py", - "metadata/parameter_reads.py", - "metadata/parameter_tree_reads.py", - "metadata/region_reads.py", - "metadata/variable_reads.py", + "metadata/reads.py", + "metadata/reads_datasets.py", + "metadata/reads_models.py", + "metadata/reads_parameter_tree.py", + "metadata/reads_parameters.py", + "metadata/reads_regions.py", + "metadata/reads_variables.py", ), ) def test_read_modules_contain_no_database_mutations(relative_path: str) -> None: diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 99e71b6a5..62827208a 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -19,7 +19,7 @@ MetadataResourceNotFoundError, UnsupportedPreviewCountryError, ) -from policyengine_api.data.v2.metadata.read_models import ( +from policyengine_api.data.v2.metadata.reads import ( MetadataCanonicalParameterValue, MetadataDataset, MetadataDatasetOption, diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index a8df1f3ac..2521d4ad1 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -555,13 +555,13 @@ def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: data_directory = Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" modules = ( data_directory / "catalog" / "catalog_selection.py", - data_directory / "metadata" / "dataset_reads.py", - data_directory / "metadata" / "model_reads.py", - data_directory / "metadata" / "parameter_reads.py", - data_directory / "metadata" / "parameter_tree_reads.py", - data_directory / "metadata" / "read_support.py", - data_directory / "metadata" / "region_reads.py", - data_directory / "metadata" / "variable_reads.py", + data_directory / "metadata" / "reads.py", + data_directory / "metadata" / "reads_datasets.py", + data_directory / "metadata" / "reads_models.py", + data_directory / "metadata" / "reads_parameter_tree.py", + data_directory / "metadata" / "reads_parameters.py", + data_directory / "metadata" / "reads_regions.py", + data_directory / "metadata" / "reads_variables.py", ) imported = set() for module in modules: @@ -595,24 +595,24 @@ def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: def test_resource_service_methods_are_defined_in_their_read_modules() -> None: expected_modules = { - "list_models": "model_reads", - "get_model": "model_reads", - "get_model_by_country": "model_reads", - "list_model_versions": "model_reads", - "get_model_version": "model_reads", - "list_variables": "variable_reads", - "get_variable": "variable_reads", - "list_parameters": "parameter_reads", - "get_parameter": "parameter_reads", - "list_parameter_children": "parameter_reads", - "list_parameter_values": "parameter_reads", - "get_parameter_value": "parameter_reads", - "list_datasets": "dataset_reads", - "get_dataset": "dataset_reads", - "list_regions": "region_reads", - "get_region": "region_reads", - "get_region_by_code": "region_reads", - "get_economy_options": "region_reads", + "list_models": "reads_models", + "get_model": "reads_models", + "get_model_by_country": "reads_models", + "list_model_versions": "reads_models", + "get_model_version": "reads_models", + "list_variables": "reads_variables", + "get_variable": "reads_variables", + "list_parameters": "reads_parameters", + "get_parameter": "reads_parameters", + "list_parameter_children": "reads_parameters", + "list_parameter_values": "reads_parameters", + "get_parameter_value": "reads_parameters", + "list_datasets": "reads_datasets", + "get_dataset": "reads_datasets", + "list_regions": "reads_regions", + "get_region": "reads_regions", + "get_region_by_code": "reads_regions", + "get_economy_options": "reads_regions", } for method_name, module_name in expected_modules.items(): From 3bb700013c7d8a82778cdff73f50536c52863ce6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:22:52 +0400 Subject: [PATCH 11/18] Refactor v2 policy and metadata layers --- .../skills/v2-code-organization.md | 197 ++++--- policyengine_api/data/v2/metadata/__init__.py | 1 - .../data/v2/metadata/reads_datasets.py | 88 ---- .../data/v2/metadata/reads_models.py | 118 ----- .../data/v2/metadata/reads_parameters.py | 243 --------- .../data/v2/metadata/reads_regions.py | 192 ------- .../data/v2/metadata/reads_variables.py | 99 ---- policyengine_api/data/v2/policies/__init__.py | 1 - policyengine_api/data/v2/policies/reads.py | 283 ---------- .../fastapi_routes/dependencies.py | 29 +- .../fastapi_routes/v2/metadata/common.py | 8 +- .../v2/metadata/geography_routes.py | 2 +- .../v2/metadata/response_models.py | 2 +- .../v2/policies/request_models.py | 4 +- .../v2/policies/response_models.py | 2 +- .../fastapi_routes/v2/policies/routes.py | 10 +- policyengine_api/services/policy_mirroring.py | 27 +- policyengine_api/services/policy_service.py | 4 +- .../services/user_policy_mirroring.py | 14 +- .../metadata/database_connectors/__init__.py | 1 + .../v2/metadata/database_connectors/reads.py | 38 ++ .../database_connectors/reads_datasets.py | 42 ++ .../reads_parameter_tree.py | 84 ++- .../database_connectors/reads_parameters.py | 112 ++++ .../database_connectors/reads_regions.py | 64 +++ .../database_connectors/reads_variables.py | 54 ++ .../services/v2/metadata/database_session.py | 23 + .../services/v2/metadata/service.py | 54 -- .../services/v2/metadata/services.py | 491 ++++++++++++++++++ .../services/v2/metadata/transformations.py | 201 +++++++ .../v2/metadata/types.py} | 106 +--- .../services/v2/metadata/validators.py | 70 +++ .../services/v2/policies/canonicalization.py | 109 ---- .../v2/policies/catalog_validation.py | 43 -- .../services/v2/policies/commands.py | 138 ----- .../services/v2/policies/creation.py | 123 ----- .../policies/database_connectors/__init__.py | 1 + .../policies/database_connectors}/creates.py | 26 +- .../v2/policies/database_connectors/reads.py | 148 ++++++ .../services/v2/policies/database_session.py | 26 + .../services/v2/policies/legacy_service.py | 132 ----- .../v2/policies/legacy_translation.py | 161 ------ .../services/v2/policies/service.py | 113 ---- .../services/v2/policies/services.py | 320 ++++++++++++ .../services/v2/policies/transformations.py | 287 ++++++++++ .../services/v2/policies/types.py | 172 ++++++ .../services/v2/policies/validators.py | 134 +++++ .../v2/user_policies/legacy_service.py | 10 +- .../v2/user_policies/legacy_translation.py | 4 +- .../services/v2/user_policies/service.py | 4 +- pyproject.toml | 2 - .../integration/test_v1_policy_dual_write.py | 17 +- tests/integration/test_v2_metadata_routes.py | 7 +- .../integration/test_v2_policy_persistence.py | 42 +- .../test_v2_user_policy_mirroring.py | 4 +- .../routes/test_policy_dual_write_routes.py | 4 +- .../test_user_policy_dual_write_routes.py | 4 +- tests/unit/services/test_policy_mirroring.py | 10 +- .../services/test_user_policy_mirroring.py | 4 +- tests/unit/v2/test_data_crud_boundaries.py | 66 ++- tests/unit/v2/test_metadata_routes.py | 16 +- tests/unit/v2/test_metadata_service.py | 81 +-- tests/unit/v2/test_policy_canonicalization.py | 10 +- tests/unit/v2/test_policy_catalog.py | 24 +- ...licy_commands.py => test_policy_inputs.py} | 40 +- .../unit/v2/test_policy_legacy_translation.py | 75 +-- .../v2/test_policy_persistence_statements.py | 2 +- tests/unit/v2/test_policy_query.py | 22 +- tests/unit/v2/test_policy_routes.py | 10 +- 69 files changed, 2628 insertions(+), 2427 deletions(-) delete mode 100644 policyengine_api/data/v2/metadata/__init__.py delete mode 100644 policyengine_api/data/v2/metadata/reads_datasets.py delete mode 100644 policyengine_api/data/v2/metadata/reads_models.py delete mode 100644 policyengine_api/data/v2/metadata/reads_parameters.py delete mode 100644 policyengine_api/data/v2/metadata/reads_regions.py delete mode 100644 policyengine_api/data/v2/metadata/reads_variables.py delete mode 100644 policyengine_api/data/v2/policies/__init__.py delete mode 100644 policyengine_api/data/v2/policies/reads.py create mode 100644 policyengine_api/services/v2/metadata/database_connectors/__init__.py create mode 100644 policyengine_api/services/v2/metadata/database_connectors/reads.py create mode 100644 policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py rename policyengine_api/{data/v2/metadata => services/v2/metadata/database_connectors}/reads_parameter_tree.py (69%) create mode 100644 policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py create mode 100644 policyengine_api/services/v2/metadata/database_connectors/reads_regions.py create mode 100644 policyengine_api/services/v2/metadata/database_connectors/reads_variables.py create mode 100644 policyengine_api/services/v2/metadata/database_session.py delete mode 100644 policyengine_api/services/v2/metadata/service.py create mode 100644 policyengine_api/services/v2/metadata/services.py create mode 100644 policyengine_api/services/v2/metadata/transformations.py rename policyengine_api/{data/v2/metadata/reads.py => services/v2/metadata/types.py} (50%) create mode 100644 policyengine_api/services/v2/metadata/validators.py delete mode 100644 policyengine_api/services/v2/policies/canonicalization.py delete mode 100644 policyengine_api/services/v2/policies/catalog_validation.py delete mode 100644 policyengine_api/services/v2/policies/commands.py delete mode 100644 policyengine_api/services/v2/policies/creation.py create mode 100644 policyengine_api/services/v2/policies/database_connectors/__init__.py rename policyengine_api/{data/v2/policies => services/v2/policies/database_connectors}/creates.py (72%) create mode 100644 policyengine_api/services/v2/policies/database_connectors/reads.py create mode 100644 policyengine_api/services/v2/policies/database_session.py delete mode 100644 policyengine_api/services/v2/policies/legacy_service.py delete mode 100644 policyengine_api/services/v2/policies/legacy_translation.py delete mode 100644 policyengine_api/services/v2/policies/service.py create mode 100644 policyengine_api/services/v2/policies/services.py create mode 100644 policyengine_api/services/v2/policies/transformations.py create mode 100644 policyengine_api/services/v2/policies/types.py create mode 100644 policyengine_api/services/v2/policies/validators.py rename tests/unit/v2/{test_policy_commands.py => test_policy_inputs.py} (73%) diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index ab08236de..330b6ec76 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -1,8 +1,8 @@ # API v2 Code Organization -API v2 resource code is divided by both resource and responsibility. Public -HTTP behavior must not be implemented in database-access modules, and database -sessions and transactions must not be opened by route modules. +Organize API v2 code first by resource and then by one explicit technical +responsibility. Route modules must not open database sessions or construct SQL. +Service modules must sequence work but must not construct or execute SQL. ## HTTP adapters @@ -23,102 +23,127 @@ metadata/ *_routes.py ``` -Request models describe HTTP request bodies. Response models describe the -public response envelope and OpenAPI output. Route modules validate HTTP-only -conditions, invoke application services, and convert typed failures to HTTP -responses. +Request models describe HTTP bodies. Response models describe public response +envelopes and OpenAPI output. Route functions handle HTTP-only conditions, +invoke one service method, and convert typed failures to HTTP responses. -## Application services +## Resource service packages -Resource-specific application code lives under `policyengine_api/services/v2/`: +New and moved API v2 resource code uses this layout: ```text -policies/ - commands.py - catalog_validation.py - canonicalization.py - creation.py - legacy_translation.py - legacy_service.py - service.py -user_policies/ - commands.py - legacy_translation.py - legacy_service.py - service.py -metadata/ - service.py +services/v2// + services.py + validators.py + transformations.py + types.py + database_session.py + database_connectors/ + __init__.py + creates.py + reads.py + updates.py + deletes.py ``` -Command models are independent of FastAPI and Flask. Native services own -request-level database sessions and transaction boundaries. Legacy translation -converts committed v1 snapshots into v2 commands. Legacy services coordinate -all work that must occur inside one Supabase transaction. Catalog validation -operates only on already-loaded records and must not execute SQL. Policy -canonicalization is deterministic application logic and must not access a -database. +Only add CRUD connector modules for operations the resource supports. Append a +specific descriptor when one operation file would become too broad, such as +`reads_variables.py` or `reads_parameter_tree.py`. -## Database access +Policies currently have `creates.py` and `reads.py` because policies are +immutable. Metadata currently has only read connector modules because metadata +routes are read-only. User-policy associations support creation, reading, +updating, and deletion; migrate that existing package to this layout when its +modules are next moved or substantially changed. -SQL reads and writes live under `policyengine_api/data/v2/`: +### `services.py` -```text -policies/ - creates.py - reads.py -user_policies/ - creates.py - reads.py - updates.py - deletes.py -metadata/ - reads.py - reads_datasets.py - reads_models.py - reads_parameter_tree.py - reads_parameters.py - reads_regions.py - reads_variables.py -``` +Define the overarching functions and classes called by route functions or +other application services. A service determines operation order, calls pure +validation and transformation functions, and passes a database session to one +or more connector functions. A service may expose an entrypoint that accepts an +existing session when several resources must change atomically in one caller- +owned transaction. + +Do not construct SQL expressions, call `Session.exec`, or define HTTP response +models in this module. + +### `validators.py` + +Define functions that inspect already-available values and either return a +validated value or raise a typed exception. Validators must not load records, +open sessions, execute SQL, mutate database rows, or construct HTTP responses. + +When validation depends on stored state, a database connector loads the +required rows and the service passes those rows to a validator. For example, +policy catalog membership is checked only after a read connector returns the +selected catalog and matching parameter identifiers. + +### `transformations.py` + +Define deterministic conversions between representations. Examples include +converting database rows to service result types, translating a detached v1 +snapshot into v2 input using already-loaded catalog records, and producing a +canonical byte representation for content deduplication. + +Transformation functions must not open sessions, execute SQL, mutate database +rows, own transaction behavior, or construct HTTP responses. + +### `types.py` + +Define framework-independent Pydantic models, dataclasses, enums, and type +aliases exchanged between routes, services, validators, transformations, and +database connectors. Names should describe the represented data, such as +`PolicyCreationInput`, not an architectural pattern such as “command.” + +Types may enforce their own field-level invariants through Pydantic validation, +but multi-record or catalog validation belongs in `validators.py`. + +### `database_session.py` + +Define the resource's database-session lifetime container. It may open and +close sessions, begin transactions, commit, or roll back. It must not construct +SQL, choose records, validate business rules, transform records into response +types, or determine multi-step operation order. + +Composition code should construct this container and inject it into the +service. A request-scoped read service may wrap one already-open session; a +write service may wrap a session factory and expose read and transaction +context managers. + +### `database_connectors/` + +Every function that constructs or executes SQL, calls `Session.get`, or mutates +ORM rows belongs in this package. Organize connector files by CRUD operation: + +- `creates.py` inserts rows or adds new ORM objects. +- `reads.py` contains `SELECT` operations and `Session.get` calls. +- `updates.py` changes existing rows. +- `deletes.py` removes rows. + +Connector functions receive a session from the service. They do not open, +commit, roll back, or close it. They do not call route functions, perform +request-level or business validation, or convert rows into public response +types. Connector modules do not call one another to sequence a workflow; the +service makes that ordering explicit. -Database-access modules are organized by SQL operation rather than by HTTP -method or table. Read modules contain every `SELECT` and `Session.get` -operation used by the resource, including reads performed while processing a -create, update, or delete request. Create modules contain inserts and ORM row -creation. Update modules modify existing rows. Delete modules remove rows. A -request may use several CRUD modules while the application service sequences -those calls and owns the transaction. - -Do not create an empty CRUD module for an operation the resource does not -support. Immutable policies therefore have only `creates.py` and `reads.py`. -Mutable user-policy associations have all four modules. Read-only metadata uses -shared read behavior and result types in `reads.py`. When that module would -become too broad, append a resource descriptor after the operation name, as in -`reads_variables.py`. Apply the same operation-first naming to future metadata -creates, updates, or deletes, and do not add modules for unsupported operations. - -Legacy mapping SQL follows the same division: mapping selection belongs in -`reads.py`, mapping insertion in `creates.py`, mapping mutation in `updates.py`, -and mapping removal in `deletes.py`. Mapping validation and retry sequencing -belong in application services, not database-access modules. The shared -`data/v2/catalog/` package remains responsible for catalog initialization, -publication, and catalog selection used by multiple resources. - -Name a database-access module for the CRUD operation it implements. Do not use `repository` as a generic synonym for SQL access. Reserve that term -for a deliberate Repository-pattern abstraction with a stable interface that -hides interchangeable persistence implementations. Direct SQL CRUD modules in -API v2 do not currently provide that abstraction. +for an intentional Repository-pattern abstraction with a stable interface that +hides interchangeable persistence implementations. API v2 currently uses +direct database connector functions. + +## Dependency direction -The ordinary request direction is: +The normal dependency direction is: ```text -route -> service -> one or more CRUD modules -> SQLModel tables +route -> service -> database session + database connectors -> SQLModel tables + -> validators + -> transformations + -> types ``` -HTTP response models may consume framework-neutral database read models. -CRUD functions may consume immutable application command models, but -database-access modules must not import route modules, perform request-level -validation, control the transaction, or construct HTTP responses. CRUD modules -must not call one another; the application service makes their ordering and -shared transaction explicit. +Database connectors may consume service-layer input types when inserting or +updating rows. They must not import route modules. Validators and +transformations may consume types and already-loaded database model instances, +but must not depend on SQLAlchemy or SQLModel query/session APIs. diff --git a/policyengine_api/data/v2/metadata/__init__.py b/policyengine_api/data/v2/metadata/__init__.py deleted file mode 100644 index 6d505f62f..000000000 --- a/policyengine_api/data/v2/metadata/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CRUD-organized database access for API v2 metadata.""" diff --git a/policyengine_api/data/v2/metadata/reads_datasets.py b/policyengine_api/data/v2/metadata/reads_datasets.py deleted file mode 100644 index 080a97fa5..000000000 --- a/policyengine_api/data/v2/metadata/reads_datasets.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Logical input-dataset metadata database reads.""" - -from __future__ import annotations - -from uuid import UUID - -from sqlmodel import col, select -from policyengine_api.data.v2.metadata.reads import ( - MetadataReadContext, - MetadataResourceNotFoundError, - page_result, - query_rows, -) -from policyengine_api.data.v2.metadata.reads import ( - MetadataDataset, - MetadataDetailResult, - MetadataPageResult, -) -from policyengine_api.data.v2.models import Dataset - - -def _dataset(dataset: Dataset) -> MetadataDataset: - return MetadataDataset( - id=dataset.id, - name=dataset.name, - description=dataset.description, - year=dataset.year, - ) - - -class DatasetReadMethods(MetadataReadContext): - """Read logical input datasets from the selected catalog.""" - - def list_datasets( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataDataset]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - rows = query_rows( - self._session, - select(Dataset) - .where( - col(Dataset.tax_benefit_model_version_id) == selected.model_version.id, - col(Dataset.is_output_dataset).is_(False), - col(Dataset.storage_path).is_(None), - ) - .order_by(col(Dataset.name)) - .offset(offset) - .limit(limit + 1), - ) - return page_result( - selected, - [_dataset(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_dataset( - self, - country_id: str, - dataset_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataDataset]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Dataset).where( - col(Dataset.id) == dataset_id, - col(Dataset.tax_benefit_model_version_id) == selected.model_version.id, - col(Dataset.is_output_dataset).is_(False), - col(Dataset.storage_path).is_(None), - ), - ) - if not rows: - raise MetadataResourceNotFoundError(f"dataset {dataset_id} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_dataset(rows[0]), - ) diff --git a/policyengine_api/data/v2/metadata/reads_models.py b/policyengine_api/data/v2/metadata/reads_models.py deleted file mode 100644 index 0eb568a37..000000000 --- a/policyengine_api/data/v2/metadata/reads_models.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Tax-benefit model and model-version metadata database reads.""" - -from __future__ import annotations - -from uuid import UUID - -from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog -from policyengine_api.data.v2.metadata.reads import ( - MetadataReadContext, - MetadataResourceNotFoundError, - page_result, -) -from policyengine_api.data.v2.metadata.reads import ( - MetadataDetailResult, - MetadataModel, - MetadataModelSelectionResult, - MetadataModelVersionDetail, - MetadataPageResult, -) - - -def _model(selected: SelectedCatalog) -> MetadataModel: - return MetadataModel( - id=selected.model.id, - name=selected.model.name, - description=selected.model_version.description, - ) - - -def _model_version(selected: SelectedCatalog) -> MetadataModelVersionDetail: - return MetadataModelVersionDetail( - id=selected.model_version.id, - model_id=selected.model.id, - version=selected.model_version.version, - description=selected.model_version.description, - current_law_id=selected.model_version.current_law_id, - metadata_time_periods=selected.model_version.metadata_time_periods, - ) - - -class ModelReadMethods(MetadataReadContext): - """Read tax-benefit models and model versions from the selected catalog.""" - - def list_models( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataModel]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - rows = [_model(selected)] if offset == 0 else [] - return page_result(selected, rows, offset=offset, limit=limit) - - def get_model( - self, - country_id: str, - model_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataModel]: - selected = self.select_catalog(country_id, policyengine_version) - if selected.model.id != model_id: - raise MetadataResourceNotFoundError(f"model {model_id} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_model(selected), - ) - - def get_model_by_country( - self, - country_id: str, - policyengine_version: str | None = None, - ) -> MetadataModelSelectionResult: - selected = self.select_catalog(country_id, policyengine_version) - return MetadataModelSelectionResult( - policyengine_version=selected.policyengine_version, - model=_model(selected), - model_version=_model_version(selected), - ) - - def list_model_versions( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataModelVersionDetail]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - rows = [_model_version(selected)] if offset == 0 else [] - return page_result(selected, rows, offset=offset, limit=limit) - - def get_model_version( - self, - country_id: str, - version_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataModelVersionDetail]: - selected = self.select_catalog(country_id, policyengine_version) - if selected.model_version.id != version_id: - raise MetadataResourceNotFoundError( - f"model version {version_id} was not found" - ) - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_model_version(selected), - ) diff --git a/policyengine_api/data/v2/metadata/reads_parameters.py b/policyengine_api/data/v2/metadata/reads_parameters.py deleted file mode 100644 index 130514a20..000000000 --- a/policyengine_api/data/v2/metadata/reads_parameters.py +++ /dev/null @@ -1,243 +0,0 @@ -"""Parameter and canonical parameter-value metadata database reads.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from uuid import UUID - -import sqlalchemy as sa -from sqlmodel import col, select -from policyengine_api.data.v2.metadata.reads_parameter_tree import ( - parameter_children_from_rows, - parameter_children_query, -) -from policyengine_api.data.v2.metadata.reads import ( - MetadataReadContext, - MetadataResourceNotFoundError, - escape_like, - page_result, - query_rows, -) -from policyengine_api.data.v2.metadata.reads import ( - MetadataCanonicalParameterValue, - MetadataDetailResult, - MetadataPageResult, - MetadataParameterChild, - MetadataParameterSummary, -) -from policyengine_api.data.v2.models import Parameter, ParameterValue - - -def _parameter(parameter: Parameter) -> MetadataParameterSummary: - return MetadataParameterSummary( - id=parameter.id, - name=parameter.name, - label=parameter.label, - description=parameter.description, - data_type=parameter.data_type, - unit=parameter.unit, - ) - - -def _parameter_value(value: ParameterValue) -> MetadataCanonicalParameterValue: - return MetadataCanonicalParameterValue( - id=value.id, - parameter_id=value.parameter_id, - value=value.value_json, - start_date=value.start_date, - end_date=value.end_date, - ) - - -def _utc_day_start(selected_time: datetime) -> datetime: - if selected_time.tzinfo is None: - selected_time = selected_time.replace(tzinfo=timezone.utc) - return selected_time.astimezone(timezone.utc).replace( - hour=0, - minute=0, - second=0, - microsecond=0, - ) - - -class ParameterReadMethods(MetadataReadContext): - """Read parameters and canonical values from the selected catalog.""" - - def list_parameters( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - search: str | None = None, - ) -> MetadataPageResult[MetadataParameterSummary]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - statement = select(Parameter).where( - col(Parameter.tax_benefit_model_version_id) == selected.model_version.id - ) - if search: - pattern = f"%{escape_like(search)}%" - statement = statement.where( - sa.or_( - col(Parameter.name).ilike(pattern, escape="\\"), - col(Parameter.label).ilike(pattern, escape="\\"), - col(Parameter.description).ilike(pattern, escape="\\"), - ) - ) - rows = query_rows( - self._session, - statement.order_by(col(Parameter.name)).offset(offset).limit(limit + 1), - ) - return page_result( - selected, - [_parameter(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_parameter( - self, - country_id: str, - parameter_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataParameterSummary]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Parameter).where( - col(Parameter.id) == parameter_id, - col(Parameter.tax_benefit_model_version_id) - == selected.model_version.id, - ), - ) - if not rows: - raise MetadataResourceNotFoundError( - f"parameter {parameter_id} was not found" - ) - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_parameter(rows[0]), - ) - - def list_parameter_children( - self, - country_id: str, - policyengine_version: str | None = None, - *, - parent_path: str = "", - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataParameterChild]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - rows = query_rows( - self._session, - parameter_children_query( - model_version_id=selected.model_version.id, - parent_path=parent_path, - dialect=self._session.get_bind().dialect.name, - offset=offset, - limit=limit, - ), - ) - return page_result( - selected, - parameter_children_from_rows(rows), - offset=offset, - limit=limit, - ) - - def list_parameter_values( - self, - country_id: str, - policyengine_version: str | None = None, - *, - parameter_id: UUID | None = None, - current: bool = False, - offset: int = 0, - limit: int = 100, - now: datetime | None = None, - ) -> MetadataPageResult[MetadataCanonicalParameterValue]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - statement = ( - select(ParameterValue) - .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) - .where( - col(Parameter.tax_benefit_model_version_id) - == selected.model_version.id, - col(ParameterValue.policy_id).is_(None), - col(ParameterValue.dynamic_id).is_(None), - ) - ) - if parameter_id is not None: - statement = statement.where( - col(ParameterValue.parameter_id) == parameter_id - ) - if current: - selected_day = _utc_day_start(now or datetime.now(timezone.utc)) - statement = statement.where( - col(ParameterValue.start_date) <= selected_day, - sa.or_( - col(ParameterValue.end_date).is_(None), - col(ParameterValue.end_date) >= selected_day, - ), - ) - rows = query_rows( - self._session, - statement.order_by( - col(Parameter.name), - col(ParameterValue.start_date).desc(), - col(ParameterValue.id), - ) - .offset(offset) - .limit(limit + 1), - ) - return page_result( - selected, - [_parameter_value(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_parameter_value( - self, - country_id: str, - value_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataCanonicalParameterValue]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(ParameterValue) - .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) - .where( - col(ParameterValue.id) == value_id, - col(Parameter.tax_benefit_model_version_id) - == selected.model_version.id, - col(ParameterValue.policy_id).is_(None), - col(ParameterValue.dynamic_id).is_(None), - ), - ) - if not rows: - raise MetadataResourceNotFoundError( - f"parameter value {value_id} was not found" - ) - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_parameter_value(rows[0]), - ) diff --git a/policyengine_api/data/v2/metadata/reads_regions.py b/policyengine_api/data/v2/metadata/reads_regions.py deleted file mode 100644 index 603476479..000000000 --- a/policyengine_api/data/v2/metadata/reads_regions.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Region and economy-option metadata database reads.""" - -from __future__ import annotations - -from uuid import UUID - -from sqlmodel import col, select - -from policyengine_api.dataset_display import get_dataset_display_label -from policyengine_api.data.v2.catalog.catalog_selection import ( - MetadataCatalogUnavailableError, -) -from policyengine_api.data.v2.metadata.reads import ( - MetadataReadContext, - MetadataResourceNotFoundError, - page_result, - query_rows, -) -from policyengine_api.data.v2.metadata.reads import ( - MetadataDatasetOption, - MetadataDetailResult, - MetadataEconomyOptionsResult, - MetadataPageResult, - MetadataRegion, - MetadataRegionOption, - MetadataRegionType, - MetadataTimePeriodOption, -) -from policyengine_api.data.v2.models import Dataset, Region - - -def _region(region: Region) -> MetadataRegion: - return MetadataRegion( - id=region.id, - code=region.code, - label=region.label, - region_type=MetadataRegionType(region.region_type.value), - requires_filter=region.requires_filter, - filter_field=region.filter_field, - filter_value=region.filter_value, - filter_strategy=region.filter_strategy, - parent_code=region.parent_code, - state_code=region.state_code, - state_name=region.state_name, - default_dataset_id=region.default_dataset_id, - ) - - -class RegionReadMethods(MetadataReadContext): - """Read regions and economy options from the selected catalog.""" - - def list_regions( - self, - country_id: str, - policyengine_version: str | None = None, - *, - region_type: str | None = None, - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataRegion]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - statement = select(Region).where( - col(Region.tax_benefit_model_version_id) == selected.model_version.id - ) - if region_type is not None: - statement = statement.where(col(Region.region_type) == region_type) - rows = query_rows( - self._session, - statement.order_by(col(Region.code)).offset(offset).limit(limit + 1), - ) - return page_result( - selected, - [_region(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_region( - self, - country_id: str, - region_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataRegion]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Region).where( - col(Region.id) == region_id, - col(Region.tax_benefit_model_version_id) == selected.model_version.id, - ), - ) - if not rows: - raise MetadataResourceNotFoundError(f"region {region_id} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_region(rows[0]), - ) - - def get_region_by_code( - self, - country_id: str, - region_code: str, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataRegion]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Region).where( - col(Region.code) == region_code, - col(Region.tax_benefit_model_version_id) == selected.model_version.id, - ), - ) - if not rows: - raise MetadataResourceNotFoundError(f"region {region_code!r} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_region(rows[0]), - ) - - def get_economy_options( - self, - country_id: str, - policyengine_version: str | None = None, - ) -> MetadataEconomyOptionsResult: - selected = self.select_catalog(country_id, policyengine_version) - regions = query_rows( - self._session, - select(Region) - .where( - col(Region.tax_benefit_model_version_id) == selected.model_version.id - ) - .order_by(col(Region.code)), - ) - national_region = next( - (region for region in regions if region.code == country_id), - None, - ) - if national_region is None: - raise MetadataCatalogUnavailableError( - f"the {country_id} national v2 region is absent" - ) - datasets = query_rows( - self._session, - select(Dataset).where( - col(Dataset.id) == national_region.default_dataset_id, - col(Dataset.tax_benefit_model_version_id) == selected.model_version.id, - col(Dataset.is_output_dataset).is_(False), - col(Dataset.storage_path).is_(None), - ), - ) - if len(datasets) != 1: - raise MetadataCatalogUnavailableError( - f"the {country_id} national v2 dataset is absent" - ) - time_periods = selected.model_version.metadata_time_periods - if ( - not isinstance(selected.model_version.current_law_id, int) - or not isinstance(time_periods, list) - or not time_periods - or any(not isinstance(year, int) for year in time_periods) - ): - raise MetadataCatalogUnavailableError( - f"the {country_id} v2 model-version options are incomplete" - ) - national_dataset = datasets[0] - return MetadataEconomyOptionsResult( - policyengine_version=selected.policyengine_version, - current_law_id=selected.model_version.current_law_id, - region=[ - MetadataRegionOption( - name=region.code, - label=region.label, - type=MetadataRegionType(region.region_type.value), - ) - for region in regions - ], - time_period=[ - MetadataTimePeriodOption(name=year, label=str(year)) - for year in time_periods - ], - datasets=[ - MetadataDatasetOption( - name=national_dataset.name, - label=get_dataset_display_label(national_dataset.name), - ) - ], - ) diff --git a/policyengine_api/data/v2/metadata/reads_variables.py b/policyengine_api/data/v2/metadata/reads_variables.py deleted file mode 100644 index b7e1ff97e..000000000 --- a/policyengine_api/data/v2/metadata/reads_variables.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Variable metadata database reads.""" - -from __future__ import annotations - -from uuid import UUID - -import sqlalchemy as sa -from sqlmodel import col, select -from policyengine_api.data.v2.metadata.reads import ( - MetadataReadContext, - MetadataResourceNotFoundError, - escape_like, - page_result, - query_rows, -) -from policyengine_api.data.v2.metadata.reads import ( - MetadataDetailResult, - MetadataPageResult, - MetadataVariable, -) -from policyengine_api.data.v2.models import Variable - - -def _variable(variable: Variable) -> MetadataVariable: - return MetadataVariable( - id=variable.id, - name=variable.name, - label=variable.label, - entity=variable.entity, - description=variable.description, - data_type=variable.data_type, - possible_values=variable.possible_values, - default_value=variable.default_value, - adds=variable.adds, - subtracts=variable.subtracts, - ) - - -class VariableReadMethods(MetadataReadContext): - """Read variables from the selected catalog.""" - - def list_variables( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - search: str | None = None, - ) -> MetadataPageResult[MetadataVariable]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - statement = select(Variable).where( - col(Variable.tax_benefit_model_version_id) == selected.model_version.id - ) - if search: - pattern = f"%{escape_like(search)}%" - statement = statement.where( - sa.or_( - col(Variable.name).ilike(pattern, escape="\\"), - col(Variable.label).ilike(pattern, escape="\\"), - col(Variable.description).ilike(pattern, escape="\\"), - ) - ) - rows = query_rows( - self._session, - statement.order_by(col(Variable.name)).offset(offset).limit(limit + 1), - ) - return page_result( - selected, - [_variable(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_variable( - self, - country_id: str, - variable_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataVariable]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Variable).where( - col(Variable.id) == variable_id, - col(Variable.tax_benefit_model_version_id) == selected.model_version.id, - ), - ) - if not rows: - raise MetadataResourceNotFoundError(f"variable {variable_id} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_variable(rows[0]), - ) diff --git a/policyengine_api/data/v2/policies/__init__.py b/policyengine_api/data/v2/policies/__init__.py deleted file mode 100644 index 0704ba052..000000000 --- a/policyengine_api/data/v2/policies/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Database create and read operations for immutable v2 policies.""" diff --git a/policyengine_api/data/v2/policies/reads.py b/policyengine_api/data/v2/policies/reads.py deleted file mode 100644 index e94f2c5a4..000000000 --- a/policyengine_api/data/v2/policies/reads.py +++ /dev/null @@ -1,283 +0,0 @@ -"""Database reads used by immutable v2 policy operations.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime -from typing import Any -from uuid import UUID - -from sqlmodel import Session, col, select - -from policyengine_api.constants import POLICYENGINE_VERSION -from policyengine_api.data.v2.catalog.catalog_selection import ( - SelectedCatalog, - select_catalog, -) -from policyengine_api.data.v2.models import ( - LegacyPolicyMapping, - Parameter, - ParameterValue, - Policy, - TaxBenefitModelVersion, -) -from policyengine_api.services.v2.policies.commands import ResolvedPolicyCreateCommand - - -class PolicyNotFoundError(LookupError): - """Raised when a policy UUID is absent from the selected country.""" - - -@dataclass(frozen=True) -class PolicyParameterValueRead: - id: UUID - parameter_id: UUID - parameter_name: str - value: Any - start_date: datetime - end_date: datetime | None - - -@dataclass(frozen=True) -class PolicyRead: - id: UUID - country_id: str - tax_benefit_model_id: UUID - tax_benefit_model_version_id: UUID - created_at: datetime - updated_at: datetime - parameter_values: tuple[PolicyParameterValueRead, ...] - - -@dataclass(frozen=True) -class PolicyPage: - items: tuple[PolicyRead, ...] - offset: int - limit: int - has_more: bool - - -def read_policy_catalog( - session: Session, - country_id: str, - *, - policyengine_version: str | None = None, - running_policyengine_version: str = POLICYENGINE_VERSION, -) -> SelectedCatalog: - """Read the exact initialized catalog selected for a policy command.""" - - return select_catalog( - session, - country_id=country_id, - running_policyengine_version=running_policyengine_version, - policyengine_version=policyengine_version, - ) - - -def read_version_parameter_ids( - session: Session, - *, - model_version_id: UUID, - requested_ids: set[UUID], -) -> set[UUID]: - """Read requested parameter IDs that belong to one model version.""" - - if not requested_ids: - return set() - return set( - session.exec( - select(Parameter.id).where( - Parameter.tax_benefit_model_version_id == model_version_id, - col(Parameter.id).in_(requested_ids), - ) - ).all() - ) - - -def read_parameters_by_name( - session: Session, - *, - model_version_id: UUID, - names: set[str], -) -> dict[str, Parameter]: - """Read named parameters that belong to one model version.""" - - if not names: - return {} - parameters = session.exec( - select(Parameter).where( - Parameter.tax_benefit_model_version_id == model_version_id, - col(Parameter.name).in_(names), - ) - ).all() - return {parameter.name: parameter for parameter in parameters} - - -def read_policy_by_content_identity( - session: Session, - *, - canonicalization_version: int, - content_hash: str, -) -> Policy | None: - """Read the policy stored under one canonical version and content hash.""" - - return session.exec( - select(Policy).where( - Policy.canonicalization_version == canonicalization_version, - Policy.content_hash == content_hash, - ) - ).one_or_none() - - -def read_stored_policy_command( - session: Session, - policy: Policy, -) -> ResolvedPolicyCreateCommand | None: - """Read stored policy content in the form used by canonicalization.""" - - model_version = session.get( - TaxBenefitModelVersion, - policy.tax_benefit_model_version_id, - ) - if model_version is None: - return None - values = session.exec( - select(ParameterValue).where(ParameterValue.policy_id == policy.id) - ).all() - return ResolvedPolicyCreateCommand.model_validate( - { - "country_id": policy.country_id, - "tax_benefit_model_id": policy.tax_benefit_model_id, - "tax_benefit_model_version_id": policy.tax_benefit_model_version_id, - "policyengine_version": model_version.version, - "parameter_values": [ - { - "parameter_id": value.parameter_id, - "value": value.value_json, - "start_date": value.start_date, - "end_date": value.end_date, - } - for value in values - ], - } - ) - - -def read_legacy_policy_mapping( - session: Session, - *, - country_id: str, - legacy_policy_id: int, - lock: bool, -) -> LegacyPolicyMapping | None: - """Read one country-scoped legacy policy mapping.""" - - statement = select(LegacyPolicyMapping).where( - LegacyPolicyMapping.country_id == country_id, - LegacyPolicyMapping.legacy_policy_id == legacy_policy_id, - ) - if lock: - statement = statement.with_for_update() - return session.exec(statement).one_or_none() - - -def _parameter_values_by_policy( - session: Session, - policy_ids: list[UUID], -) -> dict[UUID, tuple[PolicyParameterValueRead, ...]]: - grouped: dict[UUID, list[PolicyParameterValueRead]] = { - policy_id: [] for policy_id in policy_ids - } - if not policy_ids: - return {} - rows = session.exec( - select(ParameterValue, Parameter.name) - .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) - .where(col(ParameterValue.policy_id).in_(policy_ids)) - .order_by( - col(Parameter.name), - col(ParameterValue.start_date), - col(ParameterValue.id), - ) - ).all() - for value, parameter_name in rows: - if value.policy_id is None: - continue - grouped[value.policy_id].append( - PolicyParameterValueRead( - id=value.id, - parameter_id=value.parameter_id, - parameter_name=parameter_name, - value=value.value_json, - start_date=value.start_date, - end_date=value.end_date, - ) - ) - return {policy_id: tuple(values) for policy_id, values in grouped.items()} - - -def _policy_read( - policy: Policy, - values: dict[UUID, tuple[PolicyParameterValueRead, ...]], -) -> PolicyRead: - return PolicyRead( - id=policy.id, - country_id=policy.country_id, - tax_benefit_model_id=policy.tax_benefit_model_id, - tax_benefit_model_version_id=policy.tax_benefit_model_version_id, - created_at=policy.created_at, - updated_at=policy.updated_at, - parameter_values=values.get(policy.id, ()), - ) - - -def read_policy( - session: Session, - *, - country_id: str, - policy_id: UUID, -) -> PolicyRead: - """Read one complete policy only under its stored country.""" - - policy = session.exec( - select(Policy).where( - Policy.id == policy_id, - Policy.country_id == country_id, - ) - ).one_or_none() - if policy is None: - raise PolicyNotFoundError(f"policy {policy_id} was not found") - values = _parameter_values_by_policy(session, [policy.id]) - return _policy_read(policy, values) - - -def list_policies( - session: Session, - *, - country_id: str, - tax_benefit_model_id: UUID | None = None, - offset: int = 0, - limit: int = 100, -) -> PolicyPage: - """Read one deterministic bounded page with optional exact model filtering.""" - - statement = select(Policy).where(Policy.country_id == country_id) - if tax_benefit_model_id is not None: - statement = statement.where(Policy.tax_benefit_model_id == tax_benefit_model_id) - rows = session.exec( - statement.order_by(col(Policy.created_at), col(Policy.id)) - .offset(offset) - .limit(limit + 1) - ).all() - has_more = len(rows) > limit - policies = rows[:limit] - values = _parameter_values_by_policy( - session, - [policy.id for policy in policies], - ) - return PolicyPage( - items=tuple(_policy_read(policy, values) for policy in policies), - offset=offset, - limit=limit, - has_more=has_more, - ) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index b20ab0554..058d8eac4 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -13,7 +13,7 @@ from policyengine_api.json_types import JSONObject if TYPE_CHECKING: - from policyengine_api.data.v2.metadata.reads import ( + from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, MetadataDataset, MetadataDetailResult, @@ -27,15 +27,16 @@ MetadataRegion, MetadataVariable, ) - from policyengine_api.data.v2.policies.reads import PolicyPage, PolicyRead + from policyengine_api.services.v2.policies.types import ( + NativePolicyCreation, + NativePolicyCreationInput, + PolicyPage, + PolicyRead, + ) from policyengine_api.data.v2.user_policies.reads import ( UserPolicyPage, UserPolicyRead, ) - from policyengine_api.services.v2.policies.commands import ( - NativePolicyCreateCommand, - ) - from policyengine_api.services.v2.policies.service import NativePolicyCreation from policyengine_api.services.v2.user_policies.commands import ( UserPolicyCreateCommand, UserPolicyPatchCommand, @@ -206,7 +207,7 @@ class V2PolicyResourceService(Protocol): def create_policy( self, - command: "NativePolicyCreateCommand", + command: "NativePolicyCreationInput", ) -> "NativePolicyCreation": ... def get_policy(self, *, country_id: str, policy_id: UUID) -> "PolicyRead": ... @@ -299,20 +300,26 @@ def _running_policyengine_version() -> str: def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.metadata.service import V2MetadataService + from policyengine_api.services.v2.metadata.database_session import ( + MetadataDatabaseSession, + ) + from policyengine_api.services.v2.metadata.services import V2MetadataService return V2MetadataService( - get_v2_session_factory()(), + MetadataDatabaseSession(get_v2_session_factory()()), running_policyengine_version=_running_policyengine_version(), ) def _default_v2_policy_service_factory() -> V2PolicyResourceService: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.policies.service import V2PolicyService + from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, + ) + from policyengine_api.services.v2.policies.services import V2PolicyService return V2PolicyService( - get_v2_session_factory(), + PolicyDatabaseSession(get_v2_session_factory()), running_policyengine_version=_running_policyengine_version(), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index cf6fa1a3b..177f217f2 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -8,14 +8,16 @@ from pydantic import BaseModel from starlette.responses import JSONResponse -from policyengine_api.services.v2.metadata.service import ( - InvalidMetadataPageError, +from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, - MetadataResourceNotFoundError, UnsupportedPreviewCountryError, ) +from policyengine_api.services.v2.metadata.validators import ( + InvalidMetadataPageError, + MetadataResourceNotFoundError, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataErrorResponse, ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index 5cd541ab3..e3da1f81e 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Query from starlette.responses import JSONResponse -from policyengine_api.data.v2.metadata.reads import MetadataRegionType +from policyengine_api.services.v2.metadata.types import MetadataRegionType from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataDatasetDetailResponse, MetadataDatasetPageResponse, diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index 433d7426b..4be86385b 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -6,7 +6,7 @@ from pydantic import StringConstraints -from policyengine_api.data.v2.metadata.reads import ( +from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, MetadataDataset, MetadataDetailResult, diff --git a/policyengine_api/fastapi_routes/v2/policies/request_models.py b/policyengine_api/fastapi_routes/v2/policies/request_models.py index b44e9de5d..e3180b94f 100644 --- a/policyengine_api/fastapi_routes/v2/policies/request_models.py +++ b/policyengine_api/fastapi_routes/v2/policies/request_models.py @@ -1,10 +1,10 @@ """Strict HTTP request models for the native v2 policy API.""" -from policyengine_api.services.v2.policies.commands import PolicyCreateCommand +from policyengine_api.services.v2.policies.types import PolicyCreationInput MAXIMUM_POLICY_REQUEST_BYTES = 1_048_576 -class PolicyCreateRequest(PolicyCreateCommand): +class PolicyCreateRequest(PolicyCreationInput): """Native body containing immutable policy content only.""" diff --git a/policyengine_api/fastapi_routes/v2/policies/response_models.py b/policyengine_api/fastapi_routes/v2/policies/response_models.py index f1549dc73..0d3173ac4 100644 --- a/policyengine_api/fastapi_routes/v2/policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/policies/response_models.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue, StringConstraints -from policyengine_api.data.v2.policies.reads import PolicyPage, PolicyRead +from policyengine_api.services.v2.policies.types import PolicyPage, PolicyRead class StrictPolicyAPIModel(BaseModel): diff --git a/policyengine_api/fastapi_routes/v2/policies/routes.py b/policyengine_api/fastapi_routes/v2/policies/routes.py index 7260f12fb..dae1a63da 100644 --- a/policyengine_api/fastapi_routes/v2/policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/policies/routes.py @@ -27,13 +27,12 @@ PolicyPageResponse, PolicyPageResult, ) -from policyengine_api.data.v2.policies.reads import PolicyNotFoundError -from policyengine_api.services.v2.policies.catalog_validation import ( +from policyengine_api.services.v2.policies.types import NativePolicyCreationInput +from policyengine_api.services.v2.policies.validators import ( PolicyCatalogValidationError, -) -from policyengine_api.services.v2.policies.creation import ( PolicyContentHashCollisionError, PolicyCreationIntegrityError, + PolicyNotFoundError, ) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import ( @@ -46,7 +45,6 @@ PolicyCreateQuery, PolicyDetailQuery, ) -from policyengine_api.services.v2.policies.commands import NativePolicyCreateCommand class PolicyRequestTooLargeError(ValueError): @@ -148,7 +146,7 @@ def create_policy( def create() -> PolicyDetailResponse | JSONResponse: result = service_factory().create_policy( - NativePolicyCreateCommand( + NativePolicyCreationInput( **body.model_dump(), policyengine_version=query.policyengine_version, ) diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py index 1c37f2d2e..84d055b95 100644 --- a/policyengine_api/services/policy_mirroring.py +++ b/policyengine_api/services/policy_mirroring.py @@ -12,22 +12,16 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.services.v2.policies.catalog_validation import ( - PolicyCatalogValidationError, -) -from policyengine_api.services.v2.policies.creation import ( - PolicyContentHashCollisionError, - PolicyCreationIntegrityError, -) -from policyengine_api.services.v2.policies.legacy_service import ( - LegacyPolicyMappingIntegrityError, -) -from policyengine_api.services.v2.policies.legacy_service import ( +from policyengine_api.services.v2.policies.types import ( LegacyPolicyPersistenceResult, -) -from policyengine_api.services.v2.policies.legacy_translation import ( LegacyPolicySnapshot, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, LegacyPolicyTranslationError, + PolicyCatalogValidationError, + PolicyContentHashCollisionError, + PolicyCreationIntegrityError, ) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.gcp_logging import logger @@ -48,9 +42,12 @@ class PolicyMirrorUnavailableError(RuntimeError): def _default_mirror_factory() -> LegacyPolicyMirror: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.policies.service import V2PolicyService + from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, + ) + from policyengine_api.services.v2.policies.services import V2PolicyService - return V2PolicyService(get_v2_session_factory()) + return V2PolicyService(PolicyDatabaseSession(get_v2_session_factory())) def _failure_category(error: Exception) -> str: diff --git a/policyengine_api/services/policy_service.py b/policyengine_api/services/policy_service.py index e94e97ee9..32c7a8e47 100644 --- a/policyengine_api/services/policy_service.py +++ b/policyengine_api/services/policy_service.py @@ -11,9 +11,7 @@ from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import Policy -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, -) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot from policyengine_api.utils import hash_object diff --git a/policyengine_api/services/user_policy_mirroring.py b/policyengine_api/services/user_policy_mirroring.py index 5d7ce4430..7da0742fd 100644 --- a/policyengine_api/services/user_policy_mirroring.py +++ b/policyengine_api/services/user_policy_mirroring.py @@ -12,24 +12,18 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.services.v2.policies.catalog_validation import ( +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, + LegacyPolicyTranslationError, PolicyCatalogValidationError, -) -from policyengine_api.services.v2.policies.creation import ( PolicyContentHashCollisionError, PolicyCreationIntegrityError, ) -from policyengine_api.services.v2.policies.legacy_service import ( - LegacyPolicyMappingIntegrityError, -) from policyengine_api.services.v2.user_policies.legacy_service import ( LegacyUserPolicyIntegrityError, LegacyUserPolicyPersistenceResult, ) -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, - LegacyPolicyTranslationError, -) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, diff --git a/policyengine_api/services/v2/metadata/database_connectors/__init__.py b/policyengine_api/services/v2/metadata/database_connectors/__init__.py new file mode 100644 index 000000000..e1d0feec0 --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for v2 metadata services.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads.py b/policyengine_api/services/v2/metadata/database_connectors/reads.py new file mode 100644 index 000000000..d44aaa432 --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads.py @@ -0,0 +1,38 @@ +"""Shared database selections for v2 metadata operations.""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy.exc import SQLAlchemyError +from sqlmodel import Session + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + SelectedCatalog, + select_catalog, +) + + +def read_metadata_catalog( + session: Session, + *, + country_id: str, + running_policyengine_version: str, + policyengine_version: str | None, +) -> SelectedCatalog: + return select_catalog( + session, + country_id=country_id, + running_policyengine_version=running_policyengine_version, + policyengine_version=policyengine_version, + ) + + +def read_rows(session: Session, statement: Any) -> list[Any]: + try: + return list(session.exec(statement).all()) + except SQLAlchemyError as error: + raise MetadataCatalogUnavailableError( + "the v2 metadata catalog cannot be queried" + ) from error diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py new file mode 100644 index 000000000..d719e443a --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py @@ -0,0 +1,42 @@ +"""Database selections for logical input-dataset metadata.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Dataset +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + + +def read_datasets( + session: Session, *, model_version_id: UUID, offset: int, limit: int +) -> list[Dataset]: + return read_rows( + session, + select(Dataset) + .where( + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ) + .order_by(col(Dataset.name)) + .offset(offset) + .limit(limit + 1), + ) + + +def read_dataset( + session: Session, *, model_version_id: UUID, dataset_id: UUID +) -> Dataset | None: + rows = read_rows( + session, + select(Dataset).where( + col(Dataset.id) == dataset_id, + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ), + ) + return rows[0] if rows else None diff --git a/policyengine_api/data/v2/metadata/reads_parameter_tree.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py similarity index 69% rename from policyengine_api/data/v2/metadata/reads_parameter_tree.py rename to policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py index fa469f2b3..011f2d75d 100644 --- a/policyengine_api/data/v2/metadata/reads_parameter_tree.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py @@ -1,4 +1,4 @@ -"""Database reads for direct parameter-tree children.""" +"""Database selections for direct parameter-tree children.""" from __future__ import annotations @@ -6,13 +6,10 @@ from uuid import UUID import sqlalchemy as sa -from sqlmodel import col +from sqlmodel import Session, col -from policyengine_api.data.v2.metadata.reads import ( - MetadataParameterChild, - MetadataParameterSummary, -) from policyengine_api.data.v2.models import Parameter, ParameterNode +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows def _escaped_like(value: str) -> str: @@ -36,11 +33,7 @@ def _child_path(column: Any, prefix: str, dialect: str) -> Any: return sa.literal(prefix) + _path_segment(remainder, dialect) -def _direct_child_path( - column: Any, - parent_path: Any, - dialect: str, -) -> Any: +def _direct_child_path(column: Any, parent_path: Any, dialect: str) -> Any: remainder = sa.func.substr(column, sa.func.length(parent_path) + 2) return parent_path + "." + _path_segment(remainder, dialect) @@ -51,7 +44,7 @@ def _has_path_prefix(column: Any, parent_path: Any) -> Any: ) -def parameter_children_query( +def _parameter_children_statement( *, model_version_id: UUID, parent_path: str, @@ -59,8 +52,6 @@ def parameter_children_query( offset: int, limit: int, ) -> Any: - """Build one bounded query for a parameter path's direct children.""" - prefix = f"{parent_path}." if parent_path else "" escaped_prefix = _escaped_like(prefix) node_name = col(ParameterNode.name) @@ -80,24 +71,14 @@ def parameter_children_query( ), ).subquery() direct_child_paths = sa.union( - sa.select( - _direct_child_path( - node_name, - paths.c.path, - dialect, - ).label("path") - ) + sa.select(_direct_child_path(node_name, paths.c.path, dialect).label("path")) .where( node_model_version_id == model_version_id, _has_path_prefix(node_name, paths.c.path), ) .correlate(paths), sa.select( - _direct_child_path( - parameter_name, - paths.c.path, - dialect, - ).label("path") + _direct_child_path(parameter_name, paths.c.path, dialect).label("path") ) .where( parameter_model_version_id == model_version_id, @@ -116,10 +97,9 @@ def parameter_children_query( return ( sa.select( paths.c.path, - sa.func.coalesce( - col(ParameterNode.label), - col(Parameter.label), - ).label("label"), + sa.func.coalesce(col(ParameterNode.label), col(Parameter.label)).label( + "label" + ), sa.case((is_node, "node"), else_="parameter").label("type"), sa.case((is_node, direct_child_count), else_=None).label("child_count"), parameter_id.label("parameter_id"), @@ -149,28 +129,22 @@ def parameter_children_query( ) -def parameter_children_from_rows(rows: list[Any]) -> list[MetadataParameterChild]: - """Convert parameter-tree query rows into typed direct-child records.""" - - items = [] - for row in rows: - parameter = None - if row.type == "parameter": - parameter = MetadataParameterSummary( - id=row.parameter_id, - name=row.path, - label=row.parameter_label, - description=row.parameter_description, - data_type=row.parameter_data_type, - unit=row.parameter_unit, - ) - items.append( - MetadataParameterChild( - path=row.path, - label=row.label or row.path.rsplit(".", 1)[-1], - type=row.type, - child_count=row.child_count, - parameter=parameter, - ) - ) - return items +def read_parameter_children( + session: Session, + *, + model_version_id: UUID, + parent_path: str, + dialect: str, + offset: int, + limit: int, +) -> list[Any]: + return read_rows( + session, + _parameter_children_statement( + model_version_id=model_version_id, + parent_path=parent_path, + dialect=dialect, + offset=offset, + limit=limit, + ), + ) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py new file mode 100644 index 000000000..69817fb5c --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -0,0 +1,112 @@ +"""Database selections for parameter and canonical-value metadata.""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Parameter, ParameterValue +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + + +def _escape_like(value: str) -> str: + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +def read_parameters( + session: Session, + *, + model_version_id: UUID, + offset: int, + limit: int, + search: str | None, +) -> list[Parameter]: + statement = select(Parameter).where( + col(Parameter.tax_benefit_model_version_id) == model_version_id + ) + if search: + pattern = f"%{_escape_like(search)}%" + statement = statement.where( + sa.or_( + col(Parameter.name).ilike(pattern, escape="\\"), + col(Parameter.label).ilike(pattern, escape="\\"), + col(Parameter.description).ilike(pattern, escape="\\"), + ) + ) + return read_rows( + session, + statement.order_by(col(Parameter.name)).offset(offset).limit(limit + 1), + ) + + +def read_parameter( + session: Session, *, model_version_id: UUID, parameter_id: UUID +) -> Parameter | None: + rows = read_rows( + session, + select(Parameter).where( + col(Parameter.id) == parameter_id, + col(Parameter.tax_benefit_model_version_id) == model_version_id, + ), + ) + return rows[0] if rows else None + + +def read_parameter_values( + session: Session, + *, + model_version_id: UUID, + parameter_id: UUID | None, + selected_day: datetime | None, + offset: int, + limit: int, +) -> list[ParameterValue]: + statement = ( + select(ParameterValue) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), + ) + ) + if parameter_id is not None: + statement = statement.where(col(ParameterValue.parameter_id) == parameter_id) + if selected_day is not None: + statement = statement.where( + col(ParameterValue.start_date) <= selected_day, + sa.or_( + col(ParameterValue.end_date).is_(None), + col(ParameterValue.end_date) >= selected_day, + ), + ) + return read_rows( + session, + statement.order_by( + col(Parameter.name), + col(ParameterValue.start_date).desc(), + col(ParameterValue.id), + ) + .offset(offset) + .limit(limit + 1), + ) + + +def read_parameter_value( + session: Session, *, model_version_id: UUID, value_id: UUID +) -> ParameterValue | None: + rows = read_rows( + session, + select(ParameterValue) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where( + col(ParameterValue.id) == value_id, + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), + ), + ) + return rows[0] if rows else None diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_regions.py b/policyengine_api/services/v2/metadata/database_connectors/reads_regions.py new file mode 100644 index 000000000..84f85d668 --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_regions.py @@ -0,0 +1,64 @@ +"""Database selections for region and economy-option metadata.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Dataset, Region +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + + +def read_regions( + session: Session, + *, + model_version_id: UUID, + region_type: str | None = None, + offset: int | None = None, + limit: int | None = None, +) -> list[Region]: + statement = select(Region).where( + col(Region.tax_benefit_model_version_id) == model_version_id + ) + if region_type is not None: + statement = statement.where(col(Region.region_type) == region_type) + statement = statement.order_by(col(Region.code)) + if offset is not None: + statement = statement.offset(offset) + if limit is not None: + statement = statement.limit(limit + 1) + return read_rows(session, statement) + + +def read_region( + session: Session, + *, + model_version_id: UUID, + region_id: UUID | None = None, + region_code: str | None = None, +) -> Region | None: + statement = select(Region).where( + col(Region.tax_benefit_model_version_id) == model_version_id + ) + if region_id is not None: + statement = statement.where(col(Region.id) == region_id) + if region_code is not None: + statement = statement.where(col(Region.code) == region_code) + rows = read_rows(session, statement) + return rows[0] if rows else None + + +def read_input_dataset( + session: Session, *, model_version_id: UUID, dataset_id: UUID +) -> Dataset | None: + rows = read_rows( + session, + select(Dataset).where( + col(Dataset.id) == dataset_id, + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ), + ) + return rows[0] if rows else None diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py new file mode 100644 index 000000000..f9ba8dad9 --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py @@ -0,0 +1,54 @@ +"""Database selections for variable metadata.""" + +from __future__ import annotations + +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Variable +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + + +def _escape_like(value: str) -> str: + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +def read_variables( + session: Session, + *, + model_version_id: UUID, + offset: int, + limit: int, + search: str | None, +) -> list[Variable]: + statement = select(Variable).where( + col(Variable.tax_benefit_model_version_id) == model_version_id + ) + if search: + pattern = f"%{_escape_like(search)}%" + statement = statement.where( + sa.or_( + col(Variable.name).ilike(pattern, escape="\\"), + col(Variable.label).ilike(pattern, escape="\\"), + col(Variable.description).ilike(pattern, escape="\\"), + ) + ) + return read_rows( + session, + statement.order_by(col(Variable.name)).offset(offset).limit(limit + 1), + ) + + +def read_variable( + session: Session, *, model_version_id: UUID, variable_id: UUID +) -> Variable | None: + rows = read_rows( + session, + select(Variable).where( + col(Variable.id) == variable_id, + col(Variable.tax_benefit_model_version_id) == model_version_id, + ), + ) + return rows[0] if rows else None diff --git a/policyengine_api/services/v2/metadata/database_session.py b/policyengine_api/services/v2/metadata/database_session.py new file mode 100644 index 000000000..32c6f0d0d --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_session.py @@ -0,0 +1,23 @@ +"""Database session lifetime management for v2 metadata services.""" + +from __future__ import annotations + +from sqlmodel import Session + + +class MetadataDatabaseSession: + """Own the request-scoped read session used by metadata services.""" + + def __init__(self, session: Session) -> None: + self._session = session + + @property + def session(self) -> Session: + return self._session + + @property + def dialect_name(self) -> str: + return self._session.get_bind().dialect.name + + def close(self) -> None: + self._session.close() diff --git a/policyengine_api/services/v2/metadata/service.py b/policyengine_api/services/v2/metadata/service.py deleted file mode 100644 index 1c743f4a4..000000000 --- a/policyengine_api/services/v2/metadata/service.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Session-owning application service for API v2 metadata reads.""" - -from __future__ import annotations - -from policyengine_api.data.v2.catalog.catalog_selection import ( - InvalidPolicyEngineVersionError, - MetadataCatalogUnavailableError, - MetadataCatalogVersionNotFoundError, - UnsupportedPreviewCountryError, - validate_policyengine_version, -) -from policyengine_api.data.v2.metadata.reads_datasets import ( - DatasetReadMethods, -) -from policyengine_api.data.v2.metadata.reads_models import ( - ModelReadMethods, -) -from policyengine_api.data.v2.metadata.reads_parameters import ( - ParameterReadMethods, -) -from policyengine_api.data.v2.metadata.reads import ( - InvalidMetadataPageError, - MetadataResourceNotFoundError, - validate_metadata_page, -) -from policyengine_api.data.v2.metadata.reads_regions import ( - RegionReadMethods, -) -from policyengine_api.data.v2.metadata.reads_variables import ( - VariableReadMethods, -) - - -__all__ = [ - "InvalidMetadataPageError", - "InvalidPolicyEngineVersionError", - "MetadataCatalogUnavailableError", - "MetadataCatalogVersionNotFoundError", - "MetadataResourceNotFoundError", - "UnsupportedPreviewCountryError", - "V2MetadataService", - "validate_metadata_page", - "validate_policyengine_version", -] - - -class V2MetadataService( - ModelReadMethods, - VariableReadMethods, - ParameterReadMethods, - DatasetReadMethods, - RegionReadMethods, -): - """Expose resource-specific metadata read methods to the route layer.""" diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py new file mode 100644 index 000000000..1e24861f5 --- /dev/null +++ b/policyengine_api/services/v2/metadata/services.py @@ -0,0 +1,491 @@ +"""Router-facing services for API v2 metadata reads.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import UUID + +from policyengine_api.data.v2.catalog.catalog_selection import ( + SelectedCatalog, + validate_policyengine_version, +) +from policyengine_api.services.v2.metadata.database_connectors.reads import ( + read_metadata_catalog, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_datasets import ( + read_dataset, + read_datasets, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_tree import ( + read_parameter_children, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( + read_parameter, + read_parameter_value, + read_parameter_values, + read_parameters, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_regions import ( + read_input_dataset, + read_region, + read_regions, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_variables import ( + read_variable, + read_variables, +) +from policyengine_api.services.v2.metadata.database_session import ( + MetadataDatabaseSession, +) +from policyengine_api.services.v2.metadata.transformations import ( + metadata_dataset, + metadata_economy_options, + metadata_model, + metadata_model_selection, + metadata_model_version, + metadata_parameter, + metadata_parameter_children, + metadata_parameter_value, + metadata_region, + metadata_variable, + page_result, + utc_day_start, +) +from policyengine_api.services.v2.metadata.types import ( + MetadataCanonicalParameterValue, + MetadataDataset, + MetadataDetailResult, + MetadataEconomyOptionsResult, + MetadataModel, + MetadataModelSelectionResult, + MetadataModelVersionDetail, + MetadataPageResult, + MetadataParameterChild, + MetadataParameterSummary, + MetadataRegion, + MetadataVariable, +) +from policyengine_api.services.v2.metadata.validators import ( + require_metadata_resource, + validate_economy_options, + validate_metadata_page, +) + + +class V2MetadataService: + """Sequence metadata reads through a request-scoped database session.""" + + def __init__( + self, + database_session: MetadataDatabaseSession, + *, + running_policyengine_version: str, + ) -> None: + self._database_session = database_session + self._running_policyengine_version = validate_policyengine_version( + running_policyengine_version + ) + + def close(self) -> None: + self._database_session.close() + + def _select_catalog( + self, country_id: str, policyengine_version: str | None + ) -> SelectedCatalog: + return read_metadata_catalog( + self._database_session.session, + country_id=country_id, + running_policyengine_version=self._running_policyengine_version, + policyengine_version=policyengine_version, + ) + + def _select_paginated_catalog( + self, + country_id: str, + policyengine_version: str | None, + *, + offset: int, + limit: int, + ) -> SelectedCatalog: + validate_metadata_page(offset, limit) + return self._select_catalog(country_id, policyengine_version) + + def list_models( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataModel]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = [metadata_model(selected)] if offset == 0 else [] + return page_result(selected, rows, offset=offset, limit=limit) + + def get_model( + self, + country_id: str, + model_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataModel]: + selected = self._select_catalog(country_id, policyengine_version) + item = require_metadata_resource( + metadata_model(selected) if selected.model.id == model_id else None, + description=f"model {model_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, item=item + ) + + def get_model_by_country( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataModelSelectionResult: + return metadata_model_selection( + self._select_catalog(country_id, policyengine_version) + ) + + def list_model_versions( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataModelVersionDetail]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = [metadata_model_version(selected)] if offset == 0 else [] + return page_result(selected, rows, offset=offset, limit=limit) + + def get_model_version( + self, + country_id: str, + version_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataModelVersionDetail]: + selected = self._select_catalog(country_id, policyengine_version) + item = require_metadata_resource( + ( + metadata_model_version(selected) + if selected.model_version.id == version_id + else None + ), + description=f"model version {version_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, item=item + ) + + def list_variables( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + search: str | None = None, + ) -> MetadataPageResult[MetadataVariable]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_variables( + self._database_session.session, + model_version_id=selected.model_version.id, + offset=offset, + limit=limit, + search=search, + ) + return page_result( + selected, + [metadata_variable(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_variable( + self, + country_id: str, + variable_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataVariable]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_variable( + self._database_session.session, + model_version_id=selected.model_version.id, + variable_id=variable_id, + ), + description=f"variable {variable_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_variable(row), + ) + + def list_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + search: str | None = None, + ) -> MetadataPageResult[MetadataParameterSummary]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_parameters( + self._database_session.session, + model_version_id=selected.model_version.id, + offset=offset, + limit=limit, + search=search, + ) + return page_result( + selected, + [metadata_parameter(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_parameter( + self, + country_id: str, + parameter_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataParameterSummary]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_parameter( + self._database_session.session, + model_version_id=selected.model_version.id, + parameter_id=parameter_id, + ), + description=f"parameter {parameter_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_parameter(row), + ) + + def list_parameter_children( + self, + country_id: str, + policyengine_version: str | None = None, + *, + parent_path: str = "", + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataParameterChild]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_parameter_children( + self._database_session.session, + model_version_id=selected.model_version.id, + parent_path=parent_path, + dialect=self._database_session.dialect_name, + offset=offset, + limit=limit, + ) + return page_result( + selected, + metadata_parameter_children(rows), + offset=offset, + limit=limit, + ) + + def list_parameter_values( + self, + country_id: str, + policyengine_version: str | None = None, + *, + parameter_id: UUID | None = None, + current: bool = False, + offset: int = 0, + limit: int = 100, + now: datetime | None = None, + ) -> MetadataPageResult[MetadataCanonicalParameterValue]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + selected_day = ( + utc_day_start(now or datetime.now(timezone.utc)) if current else None + ) + rows = read_parameter_values( + self._database_session.session, + model_version_id=selected.model_version.id, + parameter_id=parameter_id, + selected_day=selected_day, + offset=offset, + limit=limit, + ) + return page_result( + selected, + [metadata_parameter_value(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_parameter_value( + self, + country_id: str, + value_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataCanonicalParameterValue]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_parameter_value( + self._database_session.session, + model_version_id=selected.model_version.id, + value_id=value_id, + ), + description=f"parameter value {value_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_parameter_value(row), + ) + + def list_datasets( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataDataset]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_datasets( + self._database_session.session, + model_version_id=selected.model_version.id, + offset=offset, + limit=limit, + ) + return page_result( + selected, + [metadata_dataset(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_dataset( + self, + country_id: str, + dataset_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataDataset]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_dataset( + self._database_session.session, + model_version_id=selected.model_version.id, + dataset_id=dataset_id, + ), + description=f"dataset {dataset_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_dataset(row), + ) + + def list_regions( + self, + country_id: str, + policyengine_version: str | None = None, + *, + region_type: str | None = None, + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataRegion]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_regions( + self._database_session.session, + model_version_id=selected.model_version.id, + region_type=region_type, + offset=offset, + limit=limit, + ) + return page_result( + selected, + [metadata_region(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_region( + self, + country_id: str, + region_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataRegion]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_region( + self._database_session.session, + model_version_id=selected.model_version.id, + region_id=region_id, + ), + description=f"region {region_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_region(row), + ) + + def get_region_by_code( + self, + country_id: str, + region_code: str, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataRegion]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_region( + self._database_session.session, + model_version_id=selected.model_version.id, + region_code=region_code, + ), + description=f"region {region_code!r}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_region(row), + ) + + def get_economy_options( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataEconomyOptionsResult: + selected = self._select_catalog(country_id, policyengine_version) + regions = read_regions( + self._database_session.session, + model_version_id=selected.model_version.id, + ) + national_region = next( + (region for region in regions if region.code == country_id), None + ) + national_dataset = ( + read_input_dataset( + self._database_session.session, + model_version_id=selected.model_version.id, + dataset_id=national_region.default_dataset_id, + ) + if national_region is not None + else None + ) + dataset = validate_economy_options( + selected, + country_id=country_id, + regions=regions, + national_dataset=national_dataset, + ) + return metadata_economy_options( + selected, regions=regions, national_dataset=dataset + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py new file mode 100644 index 000000000..83208f41f --- /dev/null +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -0,0 +1,201 @@ +"""Pure representation transformations for v2 metadata operations.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, TypeVar + +from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog +from policyengine_api.data.v2.models import ( + Dataset, + Parameter, + ParameterValue, + Region, + Variable, +) +from policyengine_api.dataset_display import get_dataset_display_label +from policyengine_api.services.v2.metadata.types import ( + MetadataCanonicalParameterValue, + MetadataDataset, + MetadataDatasetOption, + MetadataEconomyOptionsResult, + MetadataModel, + MetadataModelSelectionResult, + MetadataModelVersionDetail, + MetadataPageResult, + MetadataParameterChild, + MetadataParameterSummary, + MetadataRegion, + MetadataRegionOption, + MetadataRegionType, + MetadataTimePeriodOption, + MetadataVariable, +) + + +ResourceT = TypeVar("ResourceT") + + +def page_result( + selected: SelectedCatalog, + rows: list[ResourceT], + *, + offset: int, + limit: int, +) -> MetadataPageResult[ResourceT]: + return MetadataPageResult( + policyengine_version=selected.policyengine_version, + items=rows[:limit], + offset=offset, + limit=limit, + has_more=len(rows) > limit, + ) + + +def metadata_model(selected: SelectedCatalog) -> MetadataModel: + return MetadataModel( + id=selected.model.id, + name=selected.model.name, + description=selected.model_version.description, + ) + + +def metadata_model_version(selected: SelectedCatalog) -> MetadataModelVersionDetail: + return MetadataModelVersionDetail( + id=selected.model_version.id, + model_id=selected.model.id, + version=selected.model_version.version, + description=selected.model_version.description, + current_law_id=selected.model_version.current_law_id, + metadata_time_periods=selected.model_version.metadata_time_periods, + ) + + +def metadata_model_selection(selected: SelectedCatalog) -> MetadataModelSelectionResult: + return MetadataModelSelectionResult( + policyengine_version=selected.policyengine_version, + model=metadata_model(selected), + model_version=metadata_model_version(selected), + ) + + +def metadata_variable(variable: Variable) -> MetadataVariable: + return MetadataVariable( + id=variable.id, + name=variable.name, + label=variable.label, + entity=variable.entity, + description=variable.description, + data_type=variable.data_type, + possible_values=variable.possible_values, + default_value=variable.default_value, + adds=variable.adds, + subtracts=variable.subtracts, + ) + + +def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: + return MetadataParameterSummary( + id=parameter.id, + name=parameter.name, + label=parameter.label, + description=parameter.description, + data_type=parameter.data_type, + unit=parameter.unit, + ) + + +def metadata_parameter_value(value: ParameterValue) -> MetadataCanonicalParameterValue: + return MetadataCanonicalParameterValue( + id=value.id, + parameter_id=value.parameter_id, + value=value.value_json, + start_date=value.start_date, + end_date=value.end_date, + ) + + +def utc_day_start(selected_time: datetime) -> datetime: + if selected_time.tzinfo is None: + selected_time = selected_time.replace(tzinfo=timezone.utc) + return selected_time.astimezone(timezone.utc).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + + +def metadata_dataset(dataset: Dataset) -> MetadataDataset: + return MetadataDataset( + id=dataset.id, + name=dataset.name, + description=dataset.description, + year=dataset.year, + ) + + +def metadata_region(region: Region) -> MetadataRegion: + return MetadataRegion( + id=region.id, + code=region.code, + label=region.label, + region_type=MetadataRegionType(region.region_type.value), + requires_filter=region.requires_filter, + filter_field=region.filter_field, + filter_value=region.filter_value, + filter_strategy=region.filter_strategy, + parent_code=region.parent_code, + state_code=region.state_code, + state_name=region.state_name, + default_dataset_id=region.default_dataset_id, + ) + + +def metadata_parameter_children(rows: list[Any]) -> list[MetadataParameterChild]: + items = [] + for row in rows: + parameter = None + if row.type == "parameter": + parameter = MetadataParameterSummary( + id=row.parameter_id, + name=row.path, + label=row.parameter_label, + description=row.parameter_description, + data_type=row.parameter_data_type, + unit=row.parameter_unit, + ) + items.append( + MetadataParameterChild( + path=row.path, + label=row.label or row.path.rsplit(".", 1)[-1], + type=row.type, + child_count=row.child_count, + parameter=parameter, + ) + ) + return items + + +def metadata_economy_options( + selected: SelectedCatalog, *, regions: list[Region], national_dataset: Dataset +) -> MetadataEconomyOptionsResult: + return MetadataEconomyOptionsResult( + policyengine_version=selected.policyengine_version, + current_law_id=selected.model_version.current_law_id, + region=[ + MetadataRegionOption( + name=region.code, + label=region.label, + type=MetadataRegionType(region.region_type.value), + ) + for region in regions + ], + time_period=[ + MetadataTimePeriodOption(name=year, label=str(year)) + for year in selected.model_version.metadata_time_periods + ], + datasets=[ + MetadataDatasetOption( + name=national_dataset.name, + label=get_dataset_display_label(national_dataset.name), + ) + ], + ) diff --git a/policyengine_api/data/v2/metadata/reads.py b/policyengine_api/services/v2/metadata/types.py similarity index 50% rename from policyengine_api/data/v2/metadata/reads.py rename to policyengine_api/services/v2/metadata/types.py index 55beb40ee..fef5b7894 100644 --- a/policyengine_api/data/v2/metadata/reads.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -1,22 +1,13 @@ -"""Shared database reads and typed read results for API v2 metadata.""" +"""Framework-independent data exchanged by v2 metadata layers.""" from __future__ import annotations from datetime import datetime from enum import StrEnum -from typing import Any, Generic, Literal, TypeVar +from typing import Generic, Literal, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, JsonValue -from sqlalchemy.exc import SQLAlchemyError -from sqlmodel import Session - -from policyengine_api.data.v2.catalog.catalog_selection import ( - MetadataCatalogUnavailableError, - SelectedCatalog, - select_catalog as select_metadata_catalog, - validate_policyengine_version, -) class StrictResponseModel(BaseModel): @@ -159,96 +150,3 @@ class MetadataEconomyOptionsResult(StrictResponseModel): region: list[MetadataRegionOption] time_period: list[MetadataTimePeriodOption] datasets: list[MetadataDatasetOption] - - -class MetadataResourceNotFoundError(LookupError): - """Raised when a selected catalog does not contain a requested resource.""" - - -class InvalidMetadataPageError(ValueError): - """Raised when collection pagination is outside the documented bounds.""" - - -class MetadataReadContext: - """Own the session and catalog selection shared by metadata read methods.""" - - def __init__(self, session: Session, *, running_policyengine_version: str): - self._session = session - self._running_policyengine_version = validate_policyengine_version( - running_policyengine_version - ) - - def close(self) -> None: - """Close the request-owned read session.""" - - self._session.close() - - def select_catalog( - self, - country_id: str, - policyengine_version: str | None = None, - ) -> SelectedCatalog: - """Select exactly one initialized country catalog.""" - - return select_metadata_catalog( - self._session, - country_id=country_id, - running_policyengine_version=self._running_policyengine_version, - policyengine_version=policyengine_version, - ) - - def _select_paginated_catalog( - self, - country_id: str, - policyengine_version: str | None, - *, - offset: int, - limit: int, - ) -> SelectedCatalog: - validate_metadata_page(offset, limit) - return self.select_catalog(country_id, policyengine_version) - - -def page_result( - selected: SelectedCatalog, - rows: list[ResourceT], - *, - offset: int, - limit: int, -) -> MetadataPageResult[ResourceT]: - """Return one bounded response page from a limit-plus-one query.""" - - return MetadataPageResult( - policyengine_version=selected.policyengine_version, - items=rows[:limit], - offset=offset, - limit=limit, - has_more=len(rows) > limit, - ) - - -def validate_metadata_page(offset: int, limit: int) -> tuple[int, int]: - """Validate the shared v2 metadata collection bounds.""" - - if offset < 0: - raise InvalidMetadataPageError("offset must be at least 0") - if not 1 <= limit <= 500: - raise InvalidMetadataPageError("limit must be between 1 and 500") - return offset, limit - - -def query_rows(session: Session, statement: Any) -> list[Any]: - """Execute one read statement and translate database failures.""" - - try: - return list(session.exec(statement).all()) - except SQLAlchemyError as error: - raise MetadataCatalogUnavailableError( - "the v2 metadata catalog cannot be queried" - ) from error - - -def escape_like(value: str) -> str: - """Escape SQL LIKE wildcard characters in a literal search value.""" - - return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") diff --git a/policyengine_api/services/v2/metadata/validators.py b/policyengine_api/services/v2/metadata/validators.py new file mode 100644 index 000000000..1ca60c3a7 --- /dev/null +++ b/policyengine_api/services/v2/metadata/validators.py @@ -0,0 +1,70 @@ +"""Database-independent validation for v2 metadata operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, TypeVar + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, +) + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog + from policyengine_api.data.v2.models import Dataset, Region + + +class MetadataResourceNotFoundError(LookupError): + """Raised when a selected catalog does not contain a requested resource.""" + + +class InvalidMetadataPageError(ValueError): + """Raised when collection pagination is outside the documented bounds.""" + + +def validate_metadata_page(offset: int, limit: int) -> tuple[int, int]: + if offset < 0: + raise InvalidMetadataPageError("offset must be at least 0") + if not 1 <= limit <= 500: + raise InvalidMetadataPageError("limit must be between 1 and 500") + return offset, limit + + +ResourceT = TypeVar("ResourceT") + + +def require_metadata_resource( + resource: ResourceT | None, *, description: str +) -> ResourceT: + if resource is None: + raise MetadataResourceNotFoundError(f"{description} was not found") + return resource + + +def validate_economy_options( + selected: "SelectedCatalog", + *, + country_id: str, + regions: list["Region"], + national_dataset: "Dataset | None", +) -> "Dataset": + """Validate preloaded records required by the economy-options response.""" + + if not any(region.code == country_id for region in regions): + raise MetadataCatalogUnavailableError( + f"the {country_id} national v2 region is absent" + ) + if national_dataset is None: + raise MetadataCatalogUnavailableError( + f"the {country_id} national v2 dataset is absent" + ) + time_periods = selected.model_version.metadata_time_periods + if ( + not isinstance(selected.model_version.current_law_id, int) + or not isinstance(time_periods, list) + or not time_periods + or any(not isinstance(year, int) for year in time_periods) + ): + raise MetadataCatalogUnavailableError( + f"the {country_id} v2 model-version options are incomplete" + ) + return national_dataset diff --git a/policyengine_api/services/v2/policies/canonicalization.py b/policyengine_api/services/v2/policies/canonicalization.py deleted file mode 100644 index 5e45a568b..000000000 --- a/policyengine_api/services/v2/policies/canonicalization.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Database-independent content identity for immutable v2 policies.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime, timezone -from decimal import Decimal -import hashlib -import json -from typing import Any - -from policyengine_api.services.v2.policies.commands import ( - ResolvedPolicyCreateCommand, -) - - -POLICY_CANONICALIZATION_VERSION = 1 - - -@dataclass(frozen=True) -class CanonicalPolicyContent: - """Canonical bytes and SHA-256 identity for one resolved policy.""" - - version: int - document: bytes - content_hash: str - - -def _canonical_number(value: int | float) -> str: - number = Decimal(str(value)) - if number.is_zero(): - return "0" - rendered = format(number, "f") - if "." in rendered: - rendered = rendered.rstrip("0").rstrip(".") - return rendered - - -def _canonical_json(value: Any) -> str: - if value is None: - return "null" - if type(value) is bool: - return "true" if value else "false" - if type(value) in {int, float}: - return _canonical_number(value) - if type(value) is str: - return json.dumps(value, ensure_ascii=True, allow_nan=False) - if type(value) is list: - return "[" + ",".join(_canonical_json(item) for item in value) + "]" - if type(value) is dict: - members = ( - f"{json.dumps(key, ensure_ascii=True)}:{_canonical_json(value[key])}" - for key in sorted(value) - ) - return "{" + ",".join(members) + "}" - raise TypeError("canonical policy content contains a non-JSON value") - - -def canonical_utc_datetime(value: datetime) -> str: - """Render an aware datetime as fixed-width UTC with a trailing Z.""" - - utc_value = value.astimezone(timezone.utc) - return utc_value.isoformat(timespec="microseconds").replace("+00:00", "Z") - - -def canonical_policy_document(command: ResolvedPolicyCreateCommand) -> bytes: - """Serialize only immutable policy content in deterministic order.""" - - parameter_values = sorted( - command.parameter_values, - key=lambda value: ( - str(value.parameter_id), - canonical_utc_datetime(value.start_date), - "" if value.end_date is None else canonical_utc_datetime(value.end_date), - ), - ) - document = { - "canonicalization_version": POLICY_CANONICALIZATION_VERSION, - "country_id": command.country_id, - "tax_benefit_model_id": str(command.tax_benefit_model_id), - "tax_benefit_model_version_id": str(command.tax_benefit_model_version_id), - "parameter_values": [ - { - "parameter_id": str(value.parameter_id), - "value": value.value, - "start_date": canonical_utc_datetime(value.start_date), - "end_date": ( - None - if value.end_date is None - else canonical_utc_datetime(value.end_date) - ), - } - for value in parameter_values - ], - } - return _canonical_json(document).encode("ascii") - - -def canonicalize_policy( - command: ResolvedPolicyCreateCommand, -) -> CanonicalPolicyContent: - """Return versioned canonical bytes and their lowercase SHA-256 digest.""" - - document = canonical_policy_document(command) - return CanonicalPolicyContent( - version=POLICY_CANONICALIZATION_VERSION, - document=document, - content_hash=hashlib.sha256(document).hexdigest(), - ) diff --git a/policyengine_api/services/v2/policies/catalog_validation.py b/policyengine_api/services/v2/policies/catalog_validation.py deleted file mode 100644 index bdb0c12ba..000000000 --- a/policyengine_api/services/v2/policies/catalog_validation.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Database-independent catalog validation for immutable v2 policies.""" - -from __future__ import annotations - -from uuid import UUID - -from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog -from policyengine_api.services.v2.policies.commands import ( - PolicyCreateCommand, - ResolvedPolicyCreateCommand, -) - - -class PolicyCatalogValidationError(ValueError): - """Raised when policy content does not belong to the selected catalog.""" - - -def validate_policy_catalog( - command: PolicyCreateCommand, - *, - selected: SelectedCatalog, - resolved_parameter_ids: set[UUID], -) -> ResolvedPolicyCreateCommand: - """Validate preloaded catalog records and bind them to policy content.""" - - if command.tax_benefit_model_id != selected.model.id: - raise PolicyCatalogValidationError( - "tax_benefit_model_id does not match the selected country catalog" - ) - - requested_parameter_ids = {value.parameter_id for value in command.parameter_values} - if resolved_parameter_ids != requested_parameter_ids: - raise PolicyCatalogValidationError( - "every parameter_id must belong to the selected model version" - ) - - return ResolvedPolicyCreateCommand( - country_id=command.country_id, - tax_benefit_model_id=selected.model.id, - tax_benefit_model_version_id=selected.model_version.id, - policyengine_version=selected.policyengine_version, - parameter_values=command.parameter_values, - ) diff --git a/policyengine_api/services/v2/policies/commands.py b/policyengine_api/services/v2/policies/commands.py deleted file mode 100644 index ddd3d127f..000000000 --- a/policyengine_api/services/v2/policies/commands.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Route-independent commands for immutable v2 policy creation.""" - -from __future__ import annotations - -from datetime import datetime, timezone -import math -from typing import Annotated, Any -from uuid import UUID - -from pydantic import ( - AfterValidator, - BaseModel, - BeforeValidator, - ConfigDict, - Field, - JsonValue, - field_validator, - model_validator, -) - -from policyengine_api.query_parameters import CountryId, PolicyEngineVersion - - -MAXIMUM_POLICY_PARAMETER_VALUES = 1_000 -MAXIMUM_JSON_NESTING = 100 - - -def _require_json_value( - value: Any, - *, - depth: int = 0, - containers: frozenset[int] = frozenset(), -) -> Any: - if depth > MAXIMUM_JSON_NESTING: - raise ValueError("JSON values must not exceed 100 nested containers") - if value is None or type(value) in {str, bool, int}: - return value - if type(value) is float: - if not math.isfinite(value): - raise ValueError("JSON numbers must be finite") - return value - if type(value) not in {list, dict}: - raise ValueError("value must contain only standards-compliant JSON types") - identity = id(value) - if identity in containers: - raise ValueError("JSON values must not contain reference cycles") - nested_containers = containers | {identity} - if type(value) is list: - for item in value: - _require_json_value( - item, - depth=depth + 1, - containers=nested_containers, - ) - return value - for key, item in value.items(): - if type(key) is not str: - raise ValueError("JSON object keys must be strings") - _require_json_value( - item, - depth=depth + 1, - containers=nested_containers, - ) - return value - - -def _normalize_utc(value: datetime) -> datetime: - if value.tzinfo is None or value.utcoffset() is None: - raise ValueError("effective dates must include a UTC offset") - return value.astimezone(timezone.utc) - - -StrictJsonValue = Annotated[ - JsonValue, - BeforeValidator(_require_json_value), -] -UtcDateTime = Annotated[datetime, AfterValidator(_normalize_utc)] - - -class StrictPolicyCommand(BaseModel): - """Reject undeclared policy input and non-finite numeric coercion.""" - - model_config = ConfigDict(extra="forbid", allow_inf_nan=False, frozen=True) - - -class PolicyParameterValueCommand(StrictPolicyCommand): - """One normalized effective value for a catalog parameter UUID.""" - - parameter_id: UUID - value: StrictJsonValue - start_date: UtcDateTime - end_date: UtcDateTime | None = None - - @model_validator(mode="after") - def validate_period(self) -> "PolicyParameterValueCommand": - if self.end_date is not None and self.end_date < self.start_date: - raise ValueError("end_date must not precede start_date") - return self - - -class PolicyCreateCommand(StrictPolicyCommand): - """Complete immutable content accepted from native and translated inputs.""" - - country_id: CountryId - tax_benefit_model_id: UUID - parameter_values: Annotated[ - list[PolicyParameterValueCommand], - Field(max_length=MAXIMUM_POLICY_PARAMETER_VALUES), - ] - - @field_validator("parameter_values") - @classmethod - def reject_duplicate_effective_values( - cls, - values: list[PolicyParameterValueCommand], - ) -> list[PolicyParameterValueCommand]: - identities: set[tuple[UUID, datetime]] = set() - for value in values: - identity = (value.parameter_id, value.start_date) - if identity in identities: - raise ValueError( - "parameter_values must not repeat a parameter_id/start_date" - ) - identities.add(identity) - return values - - -class NativePolicyCreateCommand(PolicyCreateCommand): - """Native content plus its optional catalog-version selection.""" - - policyengine_version: PolicyEngineVersion | None = None - - -class ResolvedPolicyCreateCommand(PolicyCreateCommand): - """Validated content bound to one exact initialized catalog version.""" - - policyengine_version: PolicyEngineVersion - tax_benefit_model_version_id: UUID diff --git a/policyengine_api/services/v2/policies/creation.py b/policyengine_api/services/v2/policies/creation.py deleted file mode 100644 index 17dbf3815..000000000 --- a/policyengine_api/services/v2/policies/creation.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Application sequencing for validated immutable policy creation.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from uuid import UUID - -from sqlmodel import Session - -from policyengine_api.constants import POLICYENGINE_VERSION -from policyengine_api.data.v2.policies.creates import ( - create_parameter_values, - create_policy, -) -from policyengine_api.data.v2.policies.reads import ( - read_policy_by_content_identity, - read_policy_catalog, - read_stored_policy_command, - read_version_parameter_ids, -) -from policyengine_api.services.v2.policies.canonicalization import ( - CanonicalPolicyContent, - canonical_policy_document, - canonicalize_policy, -) -from policyengine_api.services.v2.policies.catalog_validation import ( - validate_policy_catalog, -) -from policyengine_api.services.v2.policies.commands import ( - PolicyCreateCommand, - ResolvedPolicyCreateCommand, -) - - -class PolicyCreationIntegrityError(RuntimeError): - """Raised when stored policy content cannot support safe deduplication.""" - - -class PolicyContentHashCollisionError(PolicyCreationIntegrityError): - """Raised when equal version/hash keys identify different canonical bytes.""" - - -@dataclass(frozen=True) -class PolicyCreationResult: - """New or deduplicated immutable policy identity.""" - - policy_id: UUID - created: bool - - -def resolve_policy_catalog( - session: Session, - command: PolicyCreateCommand, - *, - policyengine_version: str | None = None, - running_policyengine_version: str = POLICYENGINE_VERSION, -) -> ResolvedPolicyCreateCommand: - """Read and validate the exact catalog selected for policy creation.""" - - selected = read_policy_catalog( - session, - command.country_id, - policyengine_version=policyengine_version, - running_policyengine_version=running_policyengine_version, - ) - requested_parameter_ids = {value.parameter_id for value in command.parameter_values} - resolved_parameter_ids = read_version_parameter_ids( - session, - model_version_id=selected.model_version.id, - requested_ids=requested_parameter_ids, - ) - return validate_policy_catalog( - command, - selected=selected, - resolved_parameter_ids=resolved_parameter_ids, - ) - - -def create_resolved_policy( - session: Session, - command: ResolvedPolicyCreateCommand, - *, - canonicalizer: Callable[ - [ResolvedPolicyCreateCommand], CanonicalPolicyContent - ] = canonicalize_policy, -) -> PolicyCreationResult: - """Create one policy or verify and return equivalent stored content.""" - - content = canonicalizer(command) - created_policy_id = create_policy( - session, - command, - canonicalization_version=content.version, - content_hash=content.content_hash, - ) - if created_policy_id is not None: - create_parameter_values( - session, - policy_id=created_policy_id, - command=command, - ) - return PolicyCreationResult(policy_id=created_policy_id, created=True) - - existing = read_policy_by_content_identity( - session, - canonicalization_version=content.version, - content_hash=content.content_hash, - ) - if existing is None: - raise PolicyCreationIntegrityError( - "policy hash conflict did not resolve to a stored policy" - ) - stored_command = read_stored_policy_command(session, existing) - if stored_command is None: - raise PolicyCreationIntegrityError( - "stored policy references an absent model version" - ) - if canonical_policy_document(stored_command) != content.document: - raise PolicyContentHashCollisionError( - "stored policy content differs for the same canonical version and hash" - ) - return PolicyCreationResult(policy_id=existing.id, created=False) diff --git a/policyengine_api/services/v2/policies/database_connectors/__init__.py b/policyengine_api/services/v2/policies/database_connectors/__init__.py new file mode 100644 index 000000000..4cb277126 --- /dev/null +++ b/policyengine_api/services/v2/policies/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for v2 policy services.""" diff --git a/policyengine_api/data/v2/policies/creates.py b/policyengine_api/services/v2/policies/database_connectors/creates.py similarity index 72% rename from policyengine_api/data/v2/policies/creates.py rename to policyengine_api/services/v2/policies/database_connectors/creates.py index 204403f11..1821921b4 100644 --- a/policyengine_api/data/v2/policies/creates.py +++ b/policyengine_api/services/v2/policies/database_connectors/creates.py @@ -1,4 +1,4 @@ -"""Database creates used by immutable v2 policy operations.""" +"""Database inserts used by immutable v2 policy operations.""" from __future__ import annotations @@ -7,19 +7,13 @@ from sqlalchemy.dialects.postgresql import insert from sqlmodel import Session, col -from policyengine_api.data.v2.models import ( - LegacyPolicyMapping, - ParameterValue, - Policy, -) -from policyengine_api.services.v2.policies.commands import ( - ResolvedPolicyCreateCommand, -) +from policyengine_api.data.v2.models import LegacyPolicyMapping, ParameterValue, Policy +from policyengine_api.services.v2.policies.types import ResolvedPolicyCreationInput def create_policy( session: Session, - command: ResolvedPolicyCreateCommand, + policy_input: ResolvedPolicyCreationInput, *, canonicalization_version: int, content_hash: str, @@ -29,9 +23,9 @@ def create_policy( insert(Policy) .values( id=policy_id, - country_id=command.country_id, - tax_benefit_model_id=command.tax_benefit_model_id, - tax_benefit_model_version_id=command.tax_benefit_model_version_id, + country_id=policy_input.country_id, + tax_benefit_model_id=policy_input.tax_benefit_model_id, + tax_benefit_model_version_id=policy_input.tax_benefit_model_version_id, canonicalization_version=canonicalization_version, content_hash=content_hash, ) @@ -45,7 +39,7 @@ def create_parameter_values( session: Session, *, policy_id: UUID, - command: ResolvedPolicyCreateCommand, + policy_input: ResolvedPolicyCreationInput, ) -> None: session.add_all( [ @@ -57,7 +51,7 @@ def create_parameter_values( start_date=value.start_date, end_date=value.end_date, ) - for value in command.parameter_values + for value in policy_input.parameter_values ] ) session.flush() @@ -71,8 +65,6 @@ def create_legacy_policy_mapping( source_policy_hash: str, policy_id: UUID, ) -> UUID | None: - """Create one mapping and return its UUID, or none after a conflict.""" - return session.execute( insert(LegacyPolicyMapping) .values( diff --git a/policyengine_api/services/v2/policies/database_connectors/reads.py b/policyengine_api/services/v2/policies/database_connectors/reads.py new file mode 100644 index 000000000..0276a1b43 --- /dev/null +++ b/policyengine_api/services/v2/policies/database_connectors/reads.py @@ -0,0 +1,148 @@ +"""Database selections used by immutable v2 policy operations.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, col, select + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v2.catalog.catalog_selection import ( + SelectedCatalog, + select_catalog, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModelVersion, +) + + +def read_policy_catalog( + session: Session, + country_id: str, + *, + policyengine_version: str | None = None, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> SelectedCatalog: + return select_catalog( + session, + country_id=country_id, + running_policyengine_version=running_policyengine_version, + policyengine_version=policyengine_version, + ) + + +def read_version_parameter_ids( + session: Session, *, model_version_id: UUID, requested_ids: set[UUID] +) -> set[UUID]: + if not requested_ids: + return set() + return set( + session.exec( + select(Parameter.id).where( + Parameter.tax_benefit_model_version_id == model_version_id, + col(Parameter.id).in_(requested_ids), + ) + ).all() + ) + + +def read_parameters_by_name( + session: Session, *, model_version_id: UUID, names: set[str] +) -> dict[str, Parameter]: + if not names: + return {} + parameters = session.exec( + select(Parameter).where( + Parameter.tax_benefit_model_version_id == model_version_id, + col(Parameter.name).in_(names), + ) + ).all() + return {parameter.name: parameter for parameter in parameters} + + +def read_policy_by_content_identity( + session: Session, *, canonicalization_version: int, content_hash: str +) -> Policy | None: + return session.exec( + select(Policy).where( + Policy.canonicalization_version == canonicalization_version, + Policy.content_hash == content_hash, + ) + ).one_or_none() + + +def read_model_version( + session: Session, model_version_id: UUID +) -> TaxBenefitModelVersion | None: + return session.get(TaxBenefitModelVersion, model_version_id) + + +def read_parameter_values(session: Session, policy_id: UUID) -> list[ParameterValue]: + return list( + session.exec( + select(ParameterValue).where(ParameterValue.policy_id == policy_id) + ).all() + ) + + +def read_legacy_policy_mapping( + session: Session, *, country_id: str, legacy_policy_id: int, lock: bool +) -> LegacyPolicyMapping | None: + statement = select(LegacyPolicyMapping).where( + LegacyPolicyMapping.country_id == country_id, + LegacyPolicyMapping.legacy_policy_id == legacy_policy_id, + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def read_policy_row( + session: Session, *, country_id: str, policy_id: UUID +) -> Policy | None: + return session.exec( + select(Policy).where(Policy.id == policy_id, Policy.country_id == country_id) + ).one_or_none() + + +def read_policy_rows( + session: Session, + *, + country_id: str, + tax_benefit_model_id: UUID | None, + offset: int, + limit: int, +) -> list[Policy]: + statement = select(Policy).where(Policy.country_id == country_id) + if tax_benefit_model_id is not None: + statement = statement.where(Policy.tax_benefit_model_id == tax_benefit_model_id) + return list( + session.exec( + statement.order_by(col(Policy.created_at), col(Policy.id)) + .offset(offset) + .limit(limit + 1) + ).all() + ) + + +def read_parameter_values_with_names( + session: Session, policy_ids: list[UUID] +) -> list[tuple[ParameterValue, str]]: + if not policy_ids: + return [] + return list( + session.exec( + select(ParameterValue, Parameter.name) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where(col(ParameterValue.policy_id).in_(policy_ids)) + .order_by( + col(Parameter.name), + col(ParameterValue.start_date), + col(ParameterValue.id), + ) + ).all() + ) diff --git a/policyengine_api/services/v2/policies/database_session.py b/policyengine_api/services/v2/policies/database_session.py new file mode 100644 index 000000000..3a8a6e057 --- /dev/null +++ b/policyengine_api/services/v2/policies/database_session.py @@ -0,0 +1,26 @@ +"""Database session lifetime management for v2 policy services.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + + +class PolicyDatabaseSession: + """Open read sessions and transaction-scoped sessions for policy services.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + with self._session_factory() as session: + yield session + + @contextmanager + def transaction(self) -> Iterator[Session]: + with self._session_factory.begin() as session: + yield session diff --git a/policyengine_api/services/v2/policies/legacy_service.py b/policyengine_api/services/v2/policies/legacy_service.py deleted file mode 100644 index 4d013aee5..000000000 --- a/policyengine_api/services/v2/policies/legacy_service.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Transactional operations for mirroring committed v1 policies into v2.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass -from uuid import UUID - -from sqlmodel import Session - -from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION -from policyengine_api.data.v2.models import LegacyPolicyMapping -from policyengine_api.data.v2.policies.creates import ( - create_legacy_policy_mapping, -) -from policyengine_api.data.v2.policies.reads import ( - read_legacy_policy_mapping, -) -from policyengine_api.services.v2.policies.creation import ( - create_resolved_policy, -) -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, - translate_legacy_policy, -) - - -@dataclass(frozen=True) -class LegacyPolicyPersistenceResult: - """Destination identity and insertion outcomes for one mirror attempt.""" - - policy_id: UUID - policy_created: bool - mapping_created: bool - - -class LegacyPolicyMappingIntegrityError(RuntimeError): - """Raised when one immutable v1 identity maps inconsistently.""" - - -def verify_legacy_policy_mapping( - mapping: LegacyPolicyMapping, - *, - source_policy_hash: str, - expected_policy_id: UUID | None = None, -) -> None: - """Validate an existing mapping without performing SQL.""" - - if mapping.source_policy_hash != source_policy_hash: - raise LegacyPolicyMappingIntegrityError( - "legacy policy identity was presented with a different source hash" - ) - if expected_policy_id is not None and mapping.policy_id != expected_policy_id: - raise LegacyPolicyMappingIntegrityError( - "legacy policy mapping does not match translated immutable content" - ) - - -def persist_legacy_policy( - session: Session, - snapshot: LegacyPolicySnapshot, - *, - running_policyengine_version: str = POLICYENGINE_VERSION, - country_package_versions: Mapping[str, str] = COUNTRY_PACKAGE_VERSIONS, -) -> LegacyPolicyPersistenceResult: - """Translate, deduplicate, and map one v1 policy in the caller transaction.""" - - existing = read_legacy_policy_mapping( - session, - country_id=snapshot.country_id, - legacy_policy_id=snapshot.legacy_policy_id, - lock=True, - ) - if existing is not None: - verify_legacy_policy_mapping( - existing, - source_policy_hash=snapshot.source_policy_hash, - ) - - command = translate_legacy_policy( - session, - snapshot, - running_policyengine_version=running_policyengine_version, - country_package_versions=country_package_versions, - ) - policy_result = create_resolved_policy(session, command) - if existing is not None: - verify_legacy_policy_mapping( - existing, - source_policy_hash=snapshot.source_policy_hash, - expected_policy_id=policy_result.policy_id, - ) - return LegacyPolicyPersistenceResult( - policy_id=existing.policy_id, - policy_created=False, - mapping_created=False, - ) - - mapping_id = create_legacy_policy_mapping( - session, - country_id=snapshot.country_id, - legacy_policy_id=snapshot.legacy_policy_id, - source_policy_hash=snapshot.source_policy_hash, - policy_id=policy_result.policy_id, - ) - if mapping_id is not None: - return LegacyPolicyPersistenceResult( - policy_id=policy_result.policy_id, - policy_created=policy_result.created, - mapping_created=True, - ) - - concurrent = read_legacy_policy_mapping( - session, - country_id=snapshot.country_id, - legacy_policy_id=snapshot.legacy_policy_id, - lock=False, - ) - if concurrent is None: - raise LegacyPolicyMappingIntegrityError( - "legacy policy mapping conflict did not resolve to a stored row" - ) - verify_legacy_policy_mapping( - concurrent, - source_policy_hash=snapshot.source_policy_hash, - expected_policy_id=policy_result.policy_id, - ) - return LegacyPolicyPersistenceResult( - policy_id=concurrent.policy_id, - policy_created=False, - mapping_created=False, - ) diff --git a/policyengine_api/services/v2/policies/legacy_translation.py b/policyengine_api/services/v2/policies/legacy_translation.py deleted file mode 100644 index a597b1e96..000000000 --- a/policyengine_api/services/v2/policies/legacy_translation.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Translate committed v1 policy snapshots into v2 policy commands.""" - -from __future__ import annotations - -from collections.abc import Mapping -from datetime import date, datetime, time, timezone -from typing import Annotated - -from policyengine_core.periods import ( # type: ignore[import-untyped] - period as parse_policyengine_period, -) -from pydantic import Field, field_validator -from sqlmodel import Session - -from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION -from policyengine_api.data.v2.policies.reads import ( - read_parameters_by_name, - read_policy_catalog, -) -from policyengine_api.query_parameters import CountryId -from policyengine_api.services.v2.policies.commands import ( - PolicyCreateCommand, - PolicyParameterValueCommand, - ResolvedPolicyCreateCommand, - StrictJsonValue, - StrictPolicyCommand, -) -from policyengine_api.services.v2.policies.catalog_validation import ( - validate_policy_catalog, -) - - -class LegacyPolicyTranslationError(ValueError): - """Raised when committed v1 content cannot be interpreted exactly.""" - - -class LegacyPolicySnapshot(StrictPolicyCommand): - """Detached committed fields required by the v2 policy mirror.""" - - country_id: CountryId - legacy_policy_id: Annotated[int, Field(ge=0)] - label: Annotated[str, Field(max_length=255)] | None = None - api_version: Annotated[str, Field(min_length=1, max_length=255)] - policy_json: StrictJsonValue - source_policy_hash: Annotated[str, Field(min_length=1, max_length=255)] - - @field_validator("policy_json") - @classmethod - def require_parameter_mapping(cls, value: object) -> object: - if type(value) is not dict: - raise ValueError("legacy policy_json must be a parameter-path object") - return value - - -def _utc_midnight(value: str) -> datetime: - try: - parsed = date.fromisoformat(value) - except ValueError as error: - raise LegacyPolicyTranslationError( - f"legacy period {value!r} is invalid" - ) from error - return datetime.combine(parsed, time.min, tzinfo=timezone.utc) - - -def parse_legacy_period(value: str) -> tuple[datetime, datetime]: - """Translate one legacy period key to inclusive UTC endpoints.""" - - if not value or value != value.strip(): - raise LegacyPolicyTranslationError("legacy period must be non-empty") - if "." in value: - parts = value.split(".") - if len(parts) != 2: - raise LegacyPolicyTranslationError( - f"legacy period {value!r} must contain one date range" - ) - start_date, end_date = (_utc_midnight(item) for item in parts) - else: - try: - parsed = parse_policyengine_period(value) - start_date = _utc_midnight(str(parsed.start)) - end_date = _utc_midnight(str(parsed.stop)) - except (TypeError, ValueError) as error: - raise LegacyPolicyTranslationError( - f"legacy period {value!r} is invalid" - ) from error - if end_date < start_date: - raise LegacyPolicyTranslationError( - f"legacy period {value!r} ends before it starts" - ) - return start_date, end_date - - -def translate_legacy_policy( - session: Session, - snapshot: LegacyPolicySnapshot, - *, - running_policyengine_version: str = POLICYENGINE_VERSION, - country_package_versions: Mapping[str, str] = COUNTRY_PACKAGE_VERSIONS, -) -> ResolvedPolicyCreateCommand: - """Resolve a committed legacy reform through the exact running catalog.""" - - expected_api_version = country_package_versions.get(snapshot.country_id) - if expected_api_version is None or snapshot.api_version != expected_api_version: - raise LegacyPolicyTranslationError( - "legacy policy api_version does not match the running country package" - ) - selected = read_policy_catalog( - session, - snapshot.country_id, - running_policyengine_version=running_policyengine_version, - ) - policy_json = snapshot.policy_json - assert isinstance(policy_json, dict) - parameter_names = set(policy_json) - parameters = read_parameters_by_name( - session, - model_version_id=selected.model_version.id, - names=parameter_names, - ) - if set(parameters) != parameter_names: - raise LegacyPolicyTranslationError( - "every legacy parameter path must exist in the running catalog" - ) - - parameter_values: list[PolicyParameterValueCommand] = [] - for parameter_name in sorted(parameter_names): - period_values = policy_json[parameter_name] - if type(period_values) is not dict: - raise LegacyPolicyTranslationError( - f"legacy parameter {parameter_name!r} must contain period/value entries" - ) - for period_name, value in sorted(period_values.items()): - if type(period_name) is not str: - raise LegacyPolicyTranslationError( - f"legacy parameter {parameter_name!r} has a non-string period" - ) - start_date, end_date = parse_legacy_period(period_name) - parameter_values.append( - PolicyParameterValueCommand( - parameter_id=parameters[parameter_name].id, - value=value, - start_date=start_date, - end_date=end_date, - ) - ) - - try: - command = PolicyCreateCommand( - country_id=snapshot.country_id, - tax_benefit_model_id=selected.model.id, - parameter_values=parameter_values, - ) - except ValueError as error: - raise LegacyPolicyTranslationError( - "legacy parameter periods or values conflict" - ) from error - return validate_policy_catalog( - command, - selected=selected, - resolved_parameter_ids={parameter.id for parameter in parameters.values()}, - ) diff --git a/policyengine_api/services/v2/policies/service.py b/policyengine_api/services/v2/policies/service.py deleted file mode 100644 index eee7ce6d7..000000000 --- a/policyengine_api/services/v2/policies/service.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Session-owning application service for native and mirrored v2 policies.""" - -from __future__ import annotations - -from dataclasses import dataclass -from uuid import UUID - -from sqlalchemy.orm import sessionmaker -from sqlmodel import Session - -from policyengine_api.constants import POLICYENGINE_VERSION -from policyengine_api.data.v2.policies.reads import ( - PolicyPage, - PolicyRead, - list_policies, - read_policy, -) -from policyengine_api.services.v2.policies.commands import ( - NativePolicyCreateCommand, - PolicyCreateCommand, -) -from policyengine_api.services.v2.policies.creation import ( - create_resolved_policy, - resolve_policy_catalog, -) -from policyengine_api.services.v2.policies.legacy_service import ( - LegacyPolicyPersistenceResult, - persist_legacy_policy, -) -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, -) - - -@dataclass(frozen=True) -class NativePolicyCreation: - """Complete policy read plus whether this request inserted it.""" - - item: PolicyRead - created: bool - - -class V2PolicyService: - """Own transaction boundaries for immutable policy operations.""" - - def __init__( - self, - session_factory: sessionmaker[Session], - *, - running_policyengine_version: str = POLICYENGINE_VERSION, - ) -> None: - self._sessions = session_factory - self._running_policyengine_version = running_policyengine_version - - def create_policy( - self, - command: NativePolicyCreateCommand, - ) -> NativePolicyCreation: - content = PolicyCreateCommand.model_validate( - command.model_dump(exclude={"policyengine_version"}) - ) - with self._sessions.begin() as session: - resolved = resolve_policy_catalog( - session, - content, - policyengine_version=command.policyengine_version, - running_policyengine_version=self._running_policyengine_version, - ) - persisted = create_resolved_policy(session, resolved) - item = read_policy( - session, - country_id=command.country_id, - policy_id=persisted.policy_id, - ) - return NativePolicyCreation(item=item, created=persisted.created) - - def get_policy(self, *, country_id: str, policy_id: UUID) -> PolicyRead: - with self._sessions() as session: - return read_policy( - session, - country_id=country_id, - policy_id=policy_id, - ) - - def list_policies( - self, - *, - country_id: str, - tax_benefit_model_id: UUID | None = None, - offset: int = 0, - limit: int = 100, - ) -> PolicyPage: - with self._sessions() as session: - return list_policies( - session, - country_id=country_id, - tax_benefit_model_id=tax_benefit_model_id, - offset=offset, - limit=limit, - ) - - def mirror_legacy_policy( - self, - snapshot: LegacyPolicySnapshot, - ) -> LegacyPolicyPersistenceResult: - """Mirror one committed v1 row in one Supabase transaction.""" - - with self._sessions.begin() as session: - return persist_legacy_policy( - session, - snapshot, - running_policyengine_version=self._running_policyengine_version, - ) diff --git a/policyengine_api/services/v2/policies/services.py b/policyengine_api/services/v2/policies/services.py new file mode 100644 index 000000000..6f5ac9959 --- /dev/null +++ b/policyengine_api/services/v2/policies/services.py @@ -0,0 +1,320 @@ +"""Router-facing and legacy-mirroring services for v2 policies.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from uuid import UUID + +from sqlmodel import Session + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION +from policyengine_api.services.v2.policies.database_connectors.creates import ( + create_legacy_policy_mapping, + create_parameter_values, + create_policy, +) +from policyengine_api.services.v2.policies.database_connectors.reads import ( + read_legacy_policy_mapping, + read_model_version, + read_parameter_values, + read_parameter_values_with_names, + read_parameters_by_name, + read_policy_by_content_identity, + read_policy_catalog, + read_policy_row, + read_policy_rows, + read_version_parameter_ids, +) +from policyengine_api.services.v2.policies.database_session import PolicyDatabaseSession +from policyengine_api.services.v2.policies.transformations import ( + canonical_policy_document, + canonicalize_policy, + policy_page, + policy_parameter_values_by_policy, + policy_read, + stored_policy_creation_input, + translate_legacy_policy, +) +from policyengine_api.services.v2.policies.types import ( + CanonicalPolicyContent, + LegacyPolicyPersistenceResult, + LegacyPolicySnapshot, + NativePolicyCreation, + NativePolicyCreationInput, + PolicyCreationInput, + PolicyCreationResult, + PolicyPage, + PolicyRead, + ResolvedPolicyCreationInput, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, + PolicyContentHashCollisionError, + PolicyCreationIntegrityError, + PolicyNotFoundError, + validate_policy_catalog, + verify_legacy_policy_mapping, +) + + +def resolve_policy_creation_input( + session: Session, + policy_input: PolicyCreationInput, + *, + policyengine_version: str | None = None, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> ResolvedPolicyCreationInput: + selected = read_policy_catalog( + session, + policy_input.country_id, + policyengine_version=policyengine_version, + running_policyengine_version=running_policyengine_version, + ) + requested_parameter_ids = { + value.parameter_id for value in policy_input.parameter_values + } + resolved_parameter_ids = read_version_parameter_ids( + session, + model_version_id=selected.model_version.id, + requested_ids=requested_parameter_ids, + ) + return validate_policy_catalog( + policy_input, + selected=selected, + resolved_parameter_ids=resolved_parameter_ids, + ) + + +def create_resolved_policy( + session: Session, + policy_input: ResolvedPolicyCreationInput, + *, + canonicalizer: Callable[ + [ResolvedPolicyCreationInput], CanonicalPolicyContent + ] = canonicalize_policy, +) -> PolicyCreationResult: + content = canonicalizer(policy_input) + created_policy_id = create_policy( + session, + policy_input, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if created_policy_id is not None: + create_parameter_values( + session, + policy_id=created_policy_id, + policy_input=policy_input, + ) + return PolicyCreationResult(policy_id=created_policy_id, created=True) + + existing = read_policy_by_content_identity( + session, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if existing is None: + raise PolicyCreationIntegrityError( + "policy hash conflict did not resolve to a stored policy" + ) + model_version = read_model_version(session, existing.tax_benefit_model_version_id) + if model_version is None: + raise PolicyCreationIntegrityError( + "stored policy references an absent model version" + ) + stored_input = stored_policy_creation_input( + existing, + model_version, + read_parameter_values(session, existing.id), + ) + if canonical_policy_document(stored_input) != content.document: + raise PolicyContentHashCollisionError( + "stored policy content differs for the same canonical version and hash" + ) + return PolicyCreationResult(policy_id=existing.id, created=False) + + +def read_complete_policy( + session: Session, *, country_id: str, policy_id: UUID +) -> PolicyRead: + policy = read_policy_row(session, country_id=country_id, policy_id=policy_id) + if policy is None: + raise PolicyNotFoundError(f"policy {policy_id} was not found") + values = policy_parameter_values_by_policy( + [policy.id], read_parameter_values_with_names(session, [policy.id]) + ) + return policy_read(policy, values) + + +def read_policy_page( + session: Session, + *, + country_id: str, + tax_benefit_model_id: UUID | None, + offset: int, + limit: int, +) -> PolicyPage: + rows = read_policy_rows( + session, + country_id=country_id, + tax_benefit_model_id=tax_benefit_model_id, + offset=offset, + limit=limit, + ) + displayed_rows = rows[:limit] + policy_ids = [policy.id for policy in displayed_rows] + values = policy_parameter_values_by_policy( + policy_ids, read_parameter_values_with_names(session, policy_ids) + ) + return policy_page(rows, values, offset=offset, limit=limit) + + +def mirror_legacy_policy_in_session( + session: Session, + snapshot: LegacyPolicySnapshot, + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + country_package_versions: Mapping[str, str] = COUNTRY_PACKAGE_VERSIONS, +) -> LegacyPolicyPersistenceResult: + existing = read_legacy_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + lock=True, + ) + if existing is not None: + verify_legacy_policy_mapping( + existing, source_policy_hash=snapshot.source_policy_hash + ) + + selected = read_policy_catalog( + session, + snapshot.country_id, + running_policyengine_version=running_policyengine_version, + ) + policy_json = snapshot.policy_json + assert isinstance(policy_json, dict) + parameters = read_parameters_by_name( + session, + model_version_id=selected.model_version.id, + names=set(policy_json), + ) + policy_input = translate_legacy_policy( + snapshot, + selected=selected, + parameters=parameters, + country_package_versions=country_package_versions, + ) + policy_result = create_resolved_policy(session, policy_input) + if existing is not None: + verify_legacy_policy_mapping( + existing, + source_policy_hash=snapshot.source_policy_hash, + expected_policy_id=policy_result.policy_id, + ) + return LegacyPolicyPersistenceResult( + policy_id=existing.policy_id, + policy_created=False, + mapping_created=False, + ) + + mapping_id = create_legacy_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + source_policy_hash=snapshot.source_policy_hash, + policy_id=policy_result.policy_id, + ) + if mapping_id is not None: + return LegacyPolicyPersistenceResult( + policy_id=policy_result.policy_id, + policy_created=policy_result.created, + mapping_created=True, + ) + concurrent = read_legacy_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + lock=False, + ) + if concurrent is None: + raise LegacyPolicyMappingIntegrityError( + "legacy policy mapping conflict did not resolve to a stored row" + ) + verify_legacy_policy_mapping( + concurrent, + source_policy_hash=snapshot.source_policy_hash, + expected_policy_id=policy_result.policy_id, + ) + return LegacyPolicyPersistenceResult( + policy_id=concurrent.policy_id, + policy_created=False, + mapping_created=False, + ) + + +class V2PolicyService: + """Sequence policy operations while delegating database lifetime management.""" + + def __init__( + self, + database_session: PolicyDatabaseSession, + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + ) -> None: + self._database_session = database_session + self._running_policyengine_version = running_policyengine_version + + def create_policy( + self, policy_input: NativePolicyCreationInput + ) -> NativePolicyCreation: + content = PolicyCreationInput.model_validate( + policy_input.model_dump(exclude={"policyengine_version"}) + ) + with self._database_session.transaction() as session: + resolved = resolve_policy_creation_input( + session, + content, + policyengine_version=policy_input.policyengine_version, + running_policyengine_version=self._running_policyengine_version, + ) + persisted = create_resolved_policy(session, resolved) + item = read_complete_policy( + session, + country_id=policy_input.country_id, + policy_id=persisted.policy_id, + ) + return NativePolicyCreation(item=item, created=persisted.created) + + def get_policy(self, *, country_id: str, policy_id: UUID) -> PolicyRead: + with self._database_session.read() as session: + return read_complete_policy( + session, country_id=country_id, policy_id=policy_id + ) + + def list_policies( + self, + *, + country_id: str, + tax_benefit_model_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> PolicyPage: + with self._database_session.read() as session: + return read_policy_page( + session, + country_id=country_id, + tax_benefit_model_id=tax_benefit_model_id, + offset=offset, + limit=limit, + ) + + def mirror_legacy_policy( + self, snapshot: LegacyPolicySnapshot + ) -> LegacyPolicyPersistenceResult: + with self._database_session.transaction() as session: + return mirror_legacy_policy_in_session( + session, + snapshot, + running_policyengine_version=self._running_policyengine_version, + ) diff --git a/policyengine_api/services/v2/policies/transformations.py b/policyengine_api/services/v2/policies/transformations.py new file mode 100644 index 000000000..576ae30c8 --- /dev/null +++ b/policyengine_api/services/v2/policies/transformations.py @@ -0,0 +1,287 @@ +"""Pure representation transformations for v2 policy operations.""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import date, datetime, time, timezone +from decimal import Decimal +import hashlib +import json +from typing import Any +from uuid import UUID + +from policyengine_core.periods import period as parse_policyengine_period # type: ignore[import-untyped] + +from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog +from policyengine_api.data.v2.models import ( + Parameter, + ParameterValue, + Policy, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.policies.types import ( + CanonicalPolicyContent, + LegacyPolicySnapshot, + PolicyCreationInput, + PolicyPage, + PolicyParameterValueInput, + PolicyParameterValueRead, + PolicyRead, + ResolvedPolicyCreationInput, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyTranslationError, + validate_policy_catalog, +) + + +POLICY_CANONICALIZATION_VERSION = 1 + + +def _canonical_number(value: int | float) -> str: + number = Decimal(str(value)) + if number.is_zero(): + return "0" + rendered = format(number, "f") + if "." in rendered: + rendered = rendered.rstrip("0").rstrip(".") + return rendered + + +def _canonical_json(value: Any) -> str: + if value is None: + return "null" + if type(value) is bool: + return "true" if value else "false" + if type(value) in {int, float}: + return _canonical_number(value) + if type(value) is str: + return json.dumps(value, ensure_ascii=True, allow_nan=False) + if type(value) is list: + return "[" + ",".join(_canonical_json(item) for item in value) + "]" + if type(value) is dict: + members = ( + f"{json.dumps(key, ensure_ascii=True)}:{_canonical_json(value[key])}" + for key in sorted(value) + ) + return "{" + ",".join(members) + "}" + raise TypeError("canonical policy content contains a non-JSON value") + + +def canonical_utc_datetime(value: datetime) -> str: + utc_value = value.astimezone(timezone.utc) + return utc_value.isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def canonical_policy_document(policy_input: ResolvedPolicyCreationInput) -> bytes: + parameter_values = sorted( + policy_input.parameter_values, + key=lambda value: ( + str(value.parameter_id), + canonical_utc_datetime(value.start_date), + "" if value.end_date is None else canonical_utc_datetime(value.end_date), + ), + ) + document = { + "canonicalization_version": POLICY_CANONICALIZATION_VERSION, + "country_id": policy_input.country_id, + "tax_benefit_model_id": str(policy_input.tax_benefit_model_id), + "tax_benefit_model_version_id": str(policy_input.tax_benefit_model_version_id), + "parameter_values": [ + { + "parameter_id": str(value.parameter_id), + "value": value.value, + "start_date": canonical_utc_datetime(value.start_date), + "end_date": ( + None + if value.end_date is None + else canonical_utc_datetime(value.end_date) + ), + } + for value in parameter_values + ], + } + return _canonical_json(document).encode("ascii") + + +def canonicalize_policy( + policy_input: ResolvedPolicyCreationInput, +) -> CanonicalPolicyContent: + document = canonical_policy_document(policy_input) + return CanonicalPolicyContent( + version=POLICY_CANONICALIZATION_VERSION, + document=document, + content_hash=hashlib.sha256(document).hexdigest(), + ) + + +def stored_policy_creation_input( + policy: Policy, + model_version: TaxBenefitModelVersion, + values: list[ParameterValue], +) -> ResolvedPolicyCreationInput: + return ResolvedPolicyCreationInput.model_validate( + { + "country_id": policy.country_id, + "tax_benefit_model_id": policy.tax_benefit_model_id, + "tax_benefit_model_version_id": policy.tax_benefit_model_version_id, + "policyengine_version": model_version.version, + "parameter_values": [ + { + "parameter_id": value.parameter_id, + "value": value.value_json, + "start_date": value.start_date, + "end_date": value.end_date, + } + for value in values + ], + } + ) + + +def policy_parameter_values_by_policy( + policy_ids: list[UUID], + rows: list[tuple[ParameterValue, str]], +) -> dict[UUID, tuple[PolicyParameterValueRead, ...]]: + grouped: dict[UUID, list[PolicyParameterValueRead]] = { + policy_id: [] for policy_id in policy_ids + } + for value, parameter_name in rows: + if value.policy_id is None: + continue + grouped[value.policy_id].append( + PolicyParameterValueRead( + id=value.id, + parameter_id=value.parameter_id, + parameter_name=parameter_name, + value=value.value_json, + start_date=value.start_date, + end_date=value.end_date, + ) + ) + return {policy_id: tuple(values) for policy_id, values in grouped.items()} + + +def policy_read( + policy: Policy, + values: dict[UUID, tuple[PolicyParameterValueRead, ...]], +) -> PolicyRead: + return PolicyRead( + id=policy.id, + country_id=policy.country_id, + tax_benefit_model_id=policy.tax_benefit_model_id, + tax_benefit_model_version_id=policy.tax_benefit_model_version_id, + created_at=policy.created_at, + updated_at=policy.updated_at, + parameter_values=values.get(policy.id, ()), + ) + + +def policy_page( + rows: list[Policy], + values: dict[UUID, tuple[PolicyParameterValueRead, ...]], + *, + offset: int, + limit: int, +) -> PolicyPage: + policies = rows[:limit] + return PolicyPage( + items=tuple(policy_read(policy, values) for policy in policies), + offset=offset, + limit=limit, + has_more=len(rows) > limit, + ) + + +def _utc_midnight(value: str) -> datetime: + try: + parsed = date.fromisoformat(value) + except ValueError as error: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} is invalid" + ) from error + return datetime.combine(parsed, time.min, tzinfo=timezone.utc) + + +def parse_legacy_period(value: str) -> tuple[datetime, datetime]: + if not value or value != value.strip(): + raise LegacyPolicyTranslationError("legacy period must be non-empty") + if "." in value: + parts = value.split(".") + if len(parts) != 2: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} must contain one date range" + ) + start_date, end_date = (_utc_midnight(item) for item in parts) + else: + try: + parsed = parse_policyengine_period(value) + start_date = _utc_midnight(str(parsed.start)) + end_date = _utc_midnight(str(parsed.stop)) + except (TypeError, ValueError) as error: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} is invalid" + ) from error + if end_date < start_date: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} ends before it starts" + ) + return start_date, end_date + + +def translate_legacy_policy( + snapshot: LegacyPolicySnapshot, + *, + selected: SelectedCatalog, + parameters: Mapping[str, Parameter], + country_package_versions: Mapping[str, str], +) -> ResolvedPolicyCreationInput: + expected_api_version = country_package_versions.get(snapshot.country_id) + if expected_api_version is None or snapshot.api_version != expected_api_version: + raise LegacyPolicyTranslationError( + "legacy policy api_version does not match the running country package" + ) + policy_json = snapshot.policy_json + assert isinstance(policy_json, dict) + parameter_names = set(policy_json) + if set(parameters) != parameter_names: + raise LegacyPolicyTranslationError( + "every legacy parameter path must exist in the running catalog" + ) + + parameter_values: list[PolicyParameterValueInput] = [] + for parameter_name in sorted(parameter_names): + period_values = policy_json[parameter_name] + if type(period_values) is not dict: + raise LegacyPolicyTranslationError( + f"legacy parameter {parameter_name!r} must contain period/value entries" + ) + for period_name, value in sorted(period_values.items()): + if type(period_name) is not str: + raise LegacyPolicyTranslationError( + f"legacy parameter {parameter_name!r} has a non-string period" + ) + start_date, end_date = parse_legacy_period(period_name) + parameter_values.append( + PolicyParameterValueInput( + parameter_id=parameters[parameter_name].id, + value=value, + start_date=start_date, + end_date=end_date, + ) + ) + try: + policy_input = PolicyCreationInput( + country_id=snapshot.country_id, + tax_benefit_model_id=selected.model.id, + parameter_values=parameter_values, + ) + except ValueError as error: + raise LegacyPolicyTranslationError( + "legacy parameter periods or values conflict" + ) from error + return validate_policy_catalog( + policy_input, + selected=selected, + resolved_parameter_ids={parameter.id for parameter in parameters.values()}, + ) diff --git a/policyengine_api/services/v2/policies/types.py b/policyengine_api/services/v2/policies/types.py new file mode 100644 index 000000000..86dbc19bd --- /dev/null +++ b/policyengine_api/services/v2/policies/types.py @@ -0,0 +1,172 @@ +"""Framework-independent data exchanged by v2 policy layers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import Annotated, Any +from uuid import UUID + +from pydantic import ( + AfterValidator, + BaseModel, + BeforeValidator, + ConfigDict, + Field, + JsonValue, + field_validator, + model_validator, +) + +from policyengine_api.query_parameters import CountryId, PolicyEngineVersion +from policyengine_api.services.v2.policies.validators import ( + MAXIMUM_POLICY_PARAMETER_VALUES, + normalize_utc, + require_json_value, +) + + +StrictJsonValue = Annotated[JsonValue, BeforeValidator(require_json_value)] +UtcDateTime = Annotated[datetime, AfterValidator(normalize_utc)] + + +class StrictPolicyInput(BaseModel): + """Reject undeclared policy input and non-finite numeric coercion.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, frozen=True) + + +class PolicyParameterValueInput(StrictPolicyInput): + """One normalized effective value for a catalog parameter UUID.""" + + parameter_id: UUID + value: StrictJsonValue + start_date: UtcDateTime + end_date: UtcDateTime | None = None + + @model_validator(mode="after") + def validate_period(self) -> "PolicyParameterValueInput": + if self.end_date is not None and self.end_date < self.start_date: + raise ValueError("end_date must not precede start_date") + return self + + +class PolicyCreationInput(StrictPolicyInput): + """Complete immutable policy content accepted from any source.""" + + country_id: CountryId + tax_benefit_model_id: UUID + parameter_values: Annotated[ + list[PolicyParameterValueInput], + Field(max_length=MAXIMUM_POLICY_PARAMETER_VALUES), + ] + + @field_validator("parameter_values") + @classmethod + def reject_duplicate_effective_values( + cls, + values: list[PolicyParameterValueInput], + ) -> list[PolicyParameterValueInput]: + identities: set[tuple[UUID, datetime]] = set() + for value in values: + identity = (value.parameter_id, value.start_date) + if identity in identities: + raise ValueError( + "parameter_values must not repeat a parameter_id/start_date" + ) + identities.add(identity) + return values + + +class NativePolicyCreationInput(PolicyCreationInput): + """Native policy content plus an optional catalog-version selection.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class ResolvedPolicyCreationInput(PolicyCreationInput): + """Validated content bound to one exact initialized catalog version.""" + + policyengine_version: PolicyEngineVersion + tax_benefit_model_version_id: UUID + + +class LegacyPolicySnapshot(StrictPolicyInput): + """Detached committed fields required by the v2 policy mirror.""" + + country_id: CountryId + legacy_policy_id: Annotated[int, Field(ge=0)] + label: Annotated[str, Field(max_length=255)] | None = None + api_version: Annotated[str, Field(min_length=1, max_length=255)] + policy_json: StrictJsonValue + source_policy_hash: Annotated[str, Field(min_length=1, max_length=255)] + + @field_validator("policy_json") + @classmethod + def require_parameter_mapping(cls, value: object) -> object: + if type(value) is not dict: + raise ValueError("legacy policy_json must be a parameter-path object") + return value + + +@dataclass(frozen=True) +class CanonicalPolicyContent: + """Canonical bytes and SHA-256 identity for one resolved policy.""" + + version: int + document: bytes + content_hash: str + + +@dataclass(frozen=True) +class PolicyCreationResult: + """New or deduplicated immutable policy identity.""" + + policy_id: UUID + created: bool + + +@dataclass(frozen=True) +class LegacyPolicyPersistenceResult: + """Destination identity and insertion outcomes for one mirror attempt.""" + + policy_id: UUID + policy_created: bool + mapping_created: bool + + +@dataclass(frozen=True) +class PolicyParameterValueRead: + id: UUID + parameter_id: UUID + parameter_name: str + value: Any + start_date: datetime + end_date: datetime | None + + +@dataclass(frozen=True) +class PolicyRead: + id: UUID + country_id: str + tax_benefit_model_id: UUID + tax_benefit_model_version_id: UUID + created_at: datetime + updated_at: datetime + parameter_values: tuple[PolicyParameterValueRead, ...] + + +@dataclass(frozen=True) +class PolicyPage: + items: tuple[PolicyRead, ...] + offset: int + limit: int + has_more: bool + + +@dataclass(frozen=True) +class NativePolicyCreation: + """Complete policy read plus whether this request inserted it.""" + + item: PolicyRead + created: bool diff --git a/policyengine_api/services/v2/policies/validators.py b/policyengine_api/services/v2/policies/validators.py new file mode 100644 index 000000000..822db40b5 --- /dev/null +++ b/policyengine_api/services/v2/policies/validators.py @@ -0,0 +1,134 @@ +"""Database-independent validation for v2 policy operations.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import math +from typing import TYPE_CHECKING, Any +from uuid import UUID + +from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog +from policyengine_api.data.v2.models import LegacyPolicyMapping + +if TYPE_CHECKING: + from policyengine_api.services.v2.policies.types import ( + PolicyCreationInput, + ResolvedPolicyCreationInput, + ) + + +MAXIMUM_POLICY_PARAMETER_VALUES = 1_000 +MAXIMUM_JSON_NESTING = 100 + + +class PolicyCatalogValidationError(ValueError): + """Raised when policy content does not belong to the selected catalog.""" + + +class PolicyNotFoundError(LookupError): + """Raised when a policy UUID is absent from the selected country.""" + + +class PolicyCreationIntegrityError(RuntimeError): + """Raised when stored policy content cannot support safe deduplication.""" + + +class PolicyContentHashCollisionError(PolicyCreationIntegrityError): + """Raised when equal version/hash keys identify different canonical bytes.""" + + +class LegacyPolicyMappingIntegrityError(RuntimeError): + """Raised when one immutable v1 identity maps inconsistently.""" + + +class LegacyPolicyTranslationError(ValueError): + """Raised when committed v1 content cannot be interpreted exactly.""" + + +def require_json_value( + value: Any, + *, + depth: int = 0, + containers: frozenset[int] = frozenset(), +) -> Any: + """Reject values that cannot be represented as standards-compliant JSON.""" + + if depth > MAXIMUM_JSON_NESTING: + raise ValueError("JSON values must not exceed 100 nested containers") + if value is None or type(value) in {str, bool, int}: + return value + if type(value) is float: + if not math.isfinite(value): + raise ValueError("JSON numbers must be finite") + return value + if type(value) not in {list, dict}: + raise ValueError("value must contain only standards-compliant JSON types") + identity = id(value) + if identity in containers: + raise ValueError("JSON values must not contain reference cycles") + nested_containers = containers | {identity} + if type(value) is list: + for item in value: + require_json_value(item, depth=depth + 1, containers=nested_containers) + return value + for key, item in value.items(): + if type(key) is not str: + raise ValueError("JSON object keys must be strings") + require_json_value(item, depth=depth + 1, containers=nested_containers) + return value + + +def normalize_utc(value: datetime) -> datetime: + """Require a time-zone-aware value and normalize it to UTC.""" + + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("effective dates must include a UTC offset") + return value.astimezone(timezone.utc) + + +def validate_policy_catalog( + policy_input: "PolicyCreationInput", + *, + selected: SelectedCatalog, + resolved_parameter_ids: set[UUID], +) -> "ResolvedPolicyCreationInput": + """Validate preloaded catalog records and bind them to policy content.""" + + from policyengine_api.services.v2.policies.types import ResolvedPolicyCreationInput + + if policy_input.tax_benefit_model_id != selected.model.id: + raise PolicyCatalogValidationError( + "tax_benefit_model_id does not match the selected country catalog" + ) + requested_parameter_ids = { + value.parameter_id for value in policy_input.parameter_values + } + if resolved_parameter_ids != requested_parameter_ids: + raise PolicyCatalogValidationError( + "every parameter_id must belong to the selected model version" + ) + return ResolvedPolicyCreationInput( + country_id=policy_input.country_id, + tax_benefit_model_id=selected.model.id, + tax_benefit_model_version_id=selected.model_version.id, + policyengine_version=selected.policyengine_version, + parameter_values=policy_input.parameter_values, + ) + + +def verify_legacy_policy_mapping( + mapping: LegacyPolicyMapping, + *, + source_policy_hash: str, + expected_policy_id: UUID | None = None, +) -> None: + """Validate an existing mapping without performing SQL.""" + + if mapping.source_policy_hash != source_policy_hash: + raise LegacyPolicyMappingIntegrityError( + "legacy policy identity was presented with a different source hash" + ) + if expected_policy_id is not None and mapping.policy_id != expected_policy_id: + raise LegacyPolicyMappingIntegrityError( + "legacy policy mapping does not match translated immutable content" + ) diff --git a/policyengine_api/services/v2/user_policies/legacy_service.py b/policyengine_api/services/v2/user_policies/legacy_service.py index 0eb26e6e9..50dec4913 100644 --- a/policyengine_api/services/v2/user_policies/legacy_service.py +++ b/policyengine_api/services/v2/user_policies/legacy_service.py @@ -27,12 +27,10 @@ from policyengine_api.data.v2.user_policies.updates import ( update_legacy_user_policy_state, ) -from policyengine_api.services.v2.policies.legacy_service import ( - persist_legacy_policy, -) -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, +from policyengine_api.services.v2.policies.services import ( + mirror_legacy_policy_in_session, ) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot from policyengine_api.services.v2.user_policies.commands import ( UserPolicyCreateCommand, ) @@ -264,7 +262,7 @@ def persist_legacy_user_policy( raise LegacyUserPolicyIntegrityError( "saved policy does not reference the supplied reform snapshot" ) - policy_result = persist_legacy_policy(session, reform_snapshot) + policy_result = mirror_legacy_policy_in_session(session, reform_snapshot) user_id = resolve_legacy_user_id( session, legacy_user_id=snapshot.user_id, diff --git a/policyengine_api/services/v2/user_policies/legacy_translation.py b/policyengine_api/services/v2/user_policies/legacy_translation.py index f4352b08e..961d26c28 100644 --- a/policyengine_api/services/v2/user_policies/legacy_translation.py +++ b/policyengine_api/services/v2/user_policies/legacy_translation.py @@ -10,7 +10,7 @@ from pydantic import Field from policyengine_api.query_parameters import CountryId, LegacyUserId -from policyengine_api.services.v2.policies.commands import StrictPolicyCommand +from policyengine_api.services.v2.policies.types import StrictPolicyInput from policyengine_api.services.v2.user_policies.commands import ( UserPolicyCreateCommand, ) @@ -19,7 +19,7 @@ USER_POLICY_FINGERPRINT_VERSION = 1 -class LegacyUserPolicySnapshot(StrictPolicyCommand): +class LegacyUserPolicySnapshot(StrictPolicyInput): """Detached complete committed v1 saved-policy row.""" country_id: CountryId diff --git a/policyengine_api/services/v2/user_policies/service.py b/policyengine_api/services/v2/user_policies/service.py index 1487df6b5..7753a6fe8 100644 --- a/policyengine_api/services/v2/user_policies/service.py +++ b/policyengine_api/services/v2/user_policies/service.py @@ -26,9 +26,7 @@ from policyengine_api.data.v2.user_policies.updates import ( update_user_policy, ) -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, -) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot from policyengine_api.services.v2.user_policies.commands import ( UserPolicyCreateCommand, UserPolicyPatchCommand, diff --git a/pyproject.toml b/pyproject.toml index 2fc4fd990..c79dc0b1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,8 +81,6 @@ files = [ "policyengine_api/fastapi_routes/query_parameters.py", "policyengine_api/fastapi_routes/dependencies.py", "policyengine_api/fastapi_routes/v2", - "policyengine_api/data/v2/metadata", - "policyengine_api/data/v2/policies", "policyengine_api/data/v2/user_policies", "policyengine_api/services/v2", ] diff --git a/tests/integration/test_v1_policy_dual_write.py b/tests/integration/test_v1_policy_dual_write.py index 4b8787893..a8f4381ae 100644 --- a/tests/integration/test_v1_policy_dual_write.py +++ b/tests/integration/test_v1_policy_dual_write.py @@ -24,10 +24,13 @@ TaxBenefitModel, TaxBenefitModelVersion, ) -from policyengine_api.services.v2.policies.legacy_service import ( - persist_legacy_policy, +from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, +) +from policyengine_api.services.v2.policies.services import ( + V2PolicyService, + mirror_legacy_policy_in_session, ) -from policyengine_api.services.v2.policies.service import V2PolicyService from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from policyengine_api.services.policy_mirroring import ( PolicyMirrorUnavailableError, @@ -140,7 +143,7 @@ def test_both_commits_and_interrupted_response_retry_resolve_one_mapping() -> No model_id = None try: model_id, parameter_name = _seed_catalog(v2_sessions) - mirror_service = V2PolicyService(v2_sessions) + mirror_service = V2PolicyService(PolicyDatabaseSession(v2_sessions)) creation = _create_v1(v1_service, parameter_name) first = mirror_policy_after_commit( creation.snapshot, @@ -179,7 +182,7 @@ def test_catalog_failure_leaves_cloud_sql_committed_and_retry_completes() -> Non parameter_name = "gov.phase10.cross_database_rate" try: creation = _create_v1(v1_service, parameter_name) - mirror_service = V2PolicyService(v2_sessions) + mirror_service = V2PolicyService(PolicyDatabaseSession(v2_sessions)) with pytest.raises(PolicyMirrorUnavailableError): mirror_policy_after_commit( @@ -221,7 +224,7 @@ def test_supabase_transaction_failure_rolls_back_and_has_no_background_repair() class FailingMirror: def mirror_legacy_policy(self, snapshot): with v2_sessions.begin() as session: - persist_legacy_policy(session, snapshot) + mirror_legacy_policy_in_session(session, snapshot) raise OperationalError( "forced transaction failure", {}, @@ -252,7 +255,7 @@ def mirror_legacy_policy(self, snapshot): ) result = mirror_policy_after_commit( creation.snapshot, - mirror_factory=lambda: V2PolicyService(v2_sessions), + mirror_factory=lambda: V2PolicyService(PolicyDatabaseSession(v2_sessions)), ) assert result.mapping_created is True finally: diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index d126286d5..f1031637a 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -18,7 +18,10 @@ from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.data.v2.catalog.publication import publish_catalog -from policyengine_api.services.v2.metadata.service import V2MetadataService +from policyengine_api.services.v2.metadata.database_session import ( + MetadataDatabaseSession, +) +from policyengine_api.services.v2.metadata.services import V2MetadataService from policyengine_api.data.v2.models import ( Dataset, TaxBenefitModel, @@ -79,7 +82,7 @@ def v1_metadata(country_id: str): metadata_reader_factory=lambda: None, specification_provider=lambda: {}, v2_metadata_reader_factory=lambda: V2MetadataService( - Session(engine), + MetadataDatabaseSession(Session(engine)), running_policyengine_version=POLICYENGINE_VERSION, ), ) diff --git a/tests/integration/test_v2_policy_persistence.py b/tests/integration/test_v2_policy_persistence.py index 680f72626..e5cad92b2 100644 --- a/tests/integration/test_v2_policy_persistence.py +++ b/tests/integration/test_v2_policy_persistence.py @@ -24,26 +24,22 @@ TaxBenefitModel, TaxBenefitModelVersion, ) -from policyengine_api.services.v2.policies.canonicalization import ( - CanonicalPolicyContent, +from policyengine_api.services.v2.policies.transformations import ( canonical_policy_document, canonicalize_policy, ) -from policyengine_api.services.v2.policies.legacy_service import ( - LegacyPolicyMappingIntegrityError, -) -from policyengine_api.services.v2.policies.creation import ( - PolicyContentHashCollisionError, +from policyengine_api.services.v2.policies.services import ( create_resolved_policy, + mirror_legacy_policy_in_session, ) -from policyengine_api.services.v2.policies.commands import ( - ResolvedPolicyCreateCommand, -) -from policyengine_api.services.v2.policies.legacy_service import ( - persist_legacy_policy, -) -from policyengine_api.services.v2.policies.legacy_translation import ( +from policyengine_api.services.v2.policies.types import ( + CanonicalPolicyContent, LegacyPolicySnapshot, + ResolvedPolicyCreationInput, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, + PolicyContentHashCollisionError, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL @@ -96,8 +92,8 @@ def _command( parameter_id: UUID, *, value: object = 0.2, -) -> ResolvedPolicyCreateCommand: - return ResolvedPolicyCreateCommand( +) -> ResolvedPolicyCreationInput: + return ResolvedPolicyCreationInput( country_id="us", tax_benefit_model_id=model_id, tax_benefit_model_version_id=version_id, @@ -195,7 +191,7 @@ def test_equal_hash_with_different_canonical_bytes_is_an_integrity_error() -> No changed = _command(model_id, version_id, parameter_id, value=0.3) def simulated_collision( - command: ResolvedPolicyCreateCommand, + command: ResolvedPolicyCreationInput, ) -> CanonicalPolicyContent: return CanonicalPolicyContent( version=stored.version, @@ -250,13 +246,13 @@ def test_legacy_policy_mapping_is_many_to_one_and_retry_safe() -> None: policy_json={parameter.name: {"2026": 0.2}}, source_policy_hash="second-legacy-hash", ) - first_result = persist_legacy_policy( + first_result = mirror_legacy_policy_in_session( session, first, running_policyengine_version=version.version, country_package_versions={"us": "1.0.0"}, ) - second_result = persist_legacy_policy( + second_result = mirror_legacy_policy_in_session( session, second, running_policyengine_version=version.version, @@ -264,7 +260,7 @@ def test_legacy_policy_mapping_is_many_to_one_and_retry_safe() -> None: ) with Session(engine) as session, session.begin(): - retry = persist_legacy_policy( + retry = mirror_legacy_policy_in_session( session, first, running_policyengine_version="5.2.0", @@ -307,7 +303,7 @@ def test_changed_hash_for_one_legacy_identity_rolls_back_without_mutation() -> N policy_json={parameter_name: {"2026": 0.2}}, source_policy_hash="committed-source-hash", ) - result = persist_legacy_policy( + result = mirror_legacy_policy_in_session( session, snapshot, running_policyengine_version=version.version, @@ -322,7 +318,7 @@ def test_changed_hash_for_one_legacy_identity_rolls_back_without_mutation() -> N ) with pytest.raises(LegacyPolicyMappingIntegrityError, match="different"): with Session(engine) as session, session.begin(): - persist_legacy_policy( + mirror_legacy_policy_in_session( session, changed, running_policyengine_version="5.2.0", @@ -397,7 +393,7 @@ def test_empty_and_distinct_policy_content_persist_independently() -> None: with Session(engine) as session, session.begin(): model, version, parameter = _catalog(session) model_id = model.id - empty = ResolvedPolicyCreateCommand( + empty = ResolvedPolicyCreationInput( country_id="us", tax_benefit_model_id=model.id, tax_benefit_model_version_id=version.id, diff --git a/tests/integration/test_v2_user_policy_mirroring.py b/tests/integration/test_v2_user_policy_mirroring.py index 4ed540360..6610946f5 100644 --- a/tests/integration/test_v2_user_policy_mirroring.py +++ b/tests/integration/test_v2_user_policy_mirroring.py @@ -33,9 +33,7 @@ resolve_legacy_user_id, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, -) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot from policyengine_api.services.v2.user_policies.legacy_service import ( persist_legacy_user_policy, ) diff --git a/tests/unit/routes/test_policy_dual_write_routes.py b/tests/unit/routes/test_policy_dual_write_routes.py index 9994c131a..762a95a13 100644 --- a/tests/unit/routes/test_policy_dual_write_routes.py +++ b/tests/unit/routes/test_policy_dual_write_routes.py @@ -8,9 +8,7 @@ from flask import Flask from policyengine_api.data.v1_models import Policy -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, -) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot from policyengine_api.routes.policy_routes import policy_bp from policyengine_api.services.policy_mirroring import PolicyMirrorUnavailableError from policyengine_api.services.policy_service import PolicySetResult diff --git a/tests/unit/routes/test_user_policy_dual_write_routes.py b/tests/unit/routes/test_user_policy_dual_write_routes.py index c68e1a9ac..420f867a4 100644 --- a/tests/unit/routes/test_user_policy_dual_write_routes.py +++ b/tests/unit/routes/test_user_policy_dual_write_routes.py @@ -15,9 +15,7 @@ ) from policyengine_api.data.v1_models import UserPolicy -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, -) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, ) diff --git a/tests/unit/services/test_policy_mirroring.py b/tests/unit/services/test_policy_mirroring.py index 2232fdb6c..160b0afa5 100644 --- a/tests/unit/services/test_policy_mirroring.py +++ b/tests/unit/services/test_policy_mirroring.py @@ -8,15 +8,13 @@ import pytest from sqlalchemy.exc import OperationalError, TimeoutError -from policyengine_api.services.v2.policies.legacy_service import ( - LegacyPolicyMappingIntegrityError, -) -from policyengine_api.services.v2.policies.legacy_service import ( +from policyengine_api.services.v2.policies.types import ( LegacyPolicyPersistenceResult, -) -from policyengine_api.services.v2.policies.legacy_translation import ( LegacyPolicySnapshot, ) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, +) from policyengine_api.services.policy_mirroring import ( PolicyMirrorUnavailableError, mirror_policy_after_commit, diff --git a/tests/unit/services/test_user_policy_mirroring.py b/tests/unit/services/test_user_policy_mirroring.py index 8f19ad6d1..2b59f8bc0 100644 --- a/tests/unit/services/test_user_policy_mirroring.py +++ b/tests/unit/services/test_user_policy_mirroring.py @@ -14,9 +14,7 @@ LegacyUserPolicyIntegrityError, LegacyUserPolicyPersistenceResult, ) -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, -) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, ) diff --git a/tests/unit/v2/test_data_crud_boundaries.py b/tests/unit/v2/test_data_crud_boundaries.py index 70cdc42fe..581c2f881 100644 --- a/tests/unit/v2/test_data_crud_boundaries.py +++ b/tests/unit/v2/test_data_crud_boundaries.py @@ -1,4 +1,4 @@ -"""Structural checks for API v2 database CRUD modules.""" +"""Structural checks for API v2 service and database-connector modules.""" from __future__ import annotations @@ -10,7 +10,7 @@ PROJECT_ROOT = Path(__file__).parents[3] DATA_ROOT = PROJECT_ROOT / "policyengine_api" / "data" / "v2" -SERVICE_ROOT = PROJECT_ROOT / "policyengine_api" / "services" / "v2" / "policies" +SERVICES_ROOT = PROJECT_ROOT / "policyengine_api" / "services" / "v2" def _tree(path: Path) -> ast.Module: @@ -42,45 +42,60 @@ def _imported_modules(path: Path) -> set[str]: @pytest.mark.parametrize( "relative_path", ( - "policies/creates.py", "user_policies/creates.py", "user_policies/updates.py", "user_policies/deletes.py", ), ) -def test_mutation_modules_contain_no_database_reads(relative_path: str) -> None: +def test_existing_mutation_modules_contain_no_database_reads( + relative_path: str, +) -> None: calls = _called_names(DATA_ROOT / relative_path) assert "select" not in calls assert "get" not in calls +def test_existing_read_module_contains_no_database_mutations() -> None: + calls = _called_names(DATA_ROOT / "user_policies/reads.py") + assert calls.isdisjoint({"insert", "add", "add_all", "delete"}) + + +def test_policy_create_connectors_contain_no_database_reads() -> None: + calls = _called_names(SERVICES_ROOT / "policies/database_connectors/creates.py") + assert "select" not in calls + assert "get" not in calls + + @pytest.mark.parametrize( "relative_path", ( - "policies/reads.py", - "user_policies/reads.py", - "metadata/reads.py", - "metadata/reads_datasets.py", - "metadata/reads_models.py", - "metadata/reads_parameter_tree.py", - "metadata/reads_parameters.py", - "metadata/reads_regions.py", - "metadata/reads_variables.py", + "policies/database_connectors/reads.py", + "metadata/database_connectors/reads.py", + "metadata/database_connectors/reads_datasets.py", + "metadata/database_connectors/reads_parameter_tree.py", + "metadata/database_connectors/reads_parameters.py", + "metadata/database_connectors/reads_regions.py", + "metadata/database_connectors/reads_variables.py", ), ) -def test_read_modules_contain_no_database_mutations(relative_path: str) -> None: - calls = _called_names(DATA_ROOT / relative_path) +def test_read_connectors_contain_no_database_mutations(relative_path: str) -> None: + calls = _called_names(SERVICES_ROOT / relative_path) assert calls.isdisjoint({"insert", "add", "add_all", "delete"}) @pytest.mark.parametrize( - "filename", - ("catalog_validation.py", "canonicalization.py"), + "relative_path", + ( + "policies/validators.py", + "policies/transformations.py", + "metadata/validators.py", + "metadata/transformations.py", + ), ) -def test_policy_validation_and_identity_modules_have_no_database_dependency( - filename: str, +def test_validation_and_transformation_modules_have_no_database_query_dependency( + relative_path: str, ) -> None: - imports = _imported_modules(SERVICE_ROOT / filename) + imports = _imported_modules(SERVICES_ROOT / relative_path) assert not any( module == "sqlalchemy" or module.startswith("sqlalchemy.") @@ -88,3 +103,14 @@ def test_policy_validation_and_identity_modules_have_no_database_dependency( or module.startswith("sqlmodel.") for module in imports ) + + +@pytest.mark.parametrize( + "relative_path", + ("policies/database_session.py", "metadata/database_session.py"), +) +def test_database_session_modules_do_not_construct_queries( + relative_path: str, +) -> None: + calls = _called_names(SERVICES_ROOT / relative_path) + assert calls.isdisjoint({"select", "insert", "update", "delete", "exec"}) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 62827208a..aae650bd8 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -11,15 +11,13 @@ import pytest from policyengine_api.asgi_factory import create_asgi_app -from policyengine_api.services.v2.metadata.service import ( - InvalidMetadataPageError, +from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, - MetadataResourceNotFoundError, UnsupportedPreviewCountryError, ) -from policyengine_api.data.v2.metadata.reads import ( +from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, MetadataDataset, MetadataDatasetOption, @@ -36,6 +34,10 @@ MetadataTimePeriodOption, MetadataVariable, ) +from policyengine_api.services.v2.metadata.validators import ( + InvalidMetadataPageError, + MetadataResourceNotFoundError, +) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes import dependencies as route_dependencies from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -435,19 +437,19 @@ def test_default_reader_uses_the_installed_policyengine_version( monkeypatch: pytest.MonkeyPatch, ) -> None: from policyengine_api.data.v2 import database - from policyengine_api.services.v2.metadata import service as metadata_service + from policyengine_api.services.v2.metadata import services as metadata_services session = object() captured = {} reader = object() def query_service(candidate_session, *, running_policyengine_version): - captured["session"] = candidate_session + captured["session"] = candidate_session.session captured["version"] = running_policyengine_version return reader monkeypatch.setattr(database, "get_v2_session_factory", lambda: lambda: session) - monkeypatch.setattr(metadata_service, "V2MetadataService", query_service) + monkeypatch.setattr(metadata_services, "V2MetadataService", query_service) monkeypatch.setattr( route_dependencies.importlib_metadata, "version", diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 2521d4ad1..fe1069cb5 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -12,14 +12,19 @@ from sqlalchemy.pool import StaticPool from sqlmodel import Session, create_engine, select -from policyengine_api.services.v2.metadata.service import ( - InvalidMetadataPageError, +from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, - MetadataResourceNotFoundError, UnsupportedPreviewCountryError, - V2MetadataService, +) +from policyengine_api.services.v2.metadata.database_session import ( + MetadataDatabaseSession, +) +from policyengine_api.services.v2.metadata.services import V2MetadataService +from policyengine_api.services.v2.metadata.validators import ( + InvalidMetadataPageError, + MetadataResourceNotFoundError, ) from policyengine_api.data.v2.models import ( Dataset, @@ -203,7 +208,7 @@ def _us_model_version(session: Session) -> TaxBenefitModelVersion: def _service(session: Session) -> V2MetadataService: return V2MetadataService( - session, + MetadataDatabaseSession(session), running_policyengine_version=POLICYENGINE_VERSION, ) @@ -486,7 +491,7 @@ def test_version_selection_rejects_invalid_absent_and_unsupported_requests( service.list_variables("ca") with pytest.raises(MetadataCatalogUnavailableError): V2MetadataService( - catalog_session, + MetadataDatabaseSession(catalog_session), running_policyengine_version="4.99.0", ).list_variables("us") @@ -552,16 +557,18 @@ def test_economy_options_require_a_national_region_and_dataset( def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: - data_directory = Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" + project_package = Path(__file__).parents[3] / "policyengine_api" + connector_directory = ( + project_package / "services" / "v2" / "metadata" / "database_connectors" + ) modules = ( - data_directory / "catalog" / "catalog_selection.py", - data_directory / "metadata" / "reads.py", - data_directory / "metadata" / "reads_datasets.py", - data_directory / "metadata" / "reads_models.py", - data_directory / "metadata" / "reads_parameter_tree.py", - data_directory / "metadata" / "reads_parameters.py", - data_directory / "metadata" / "reads_regions.py", - data_directory / "metadata" / "reads_variables.py", + project_package / "data" / "v2" / "catalog" / "catalog_selection.py", + connector_directory / "reads.py", + connector_directory / "reads_datasets.py", + connector_directory / "reads_parameter_tree.py", + connector_directory / "reads_parameters.py", + connector_directory / "reads_regions.py", + connector_directory / "reads_variables.py", ) imported = set() for module in modules: @@ -593,28 +600,28 @@ def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: ) -def test_resource_service_methods_are_defined_in_their_read_modules() -> None: - expected_modules = { - "list_models": "reads_models", - "get_model": "reads_models", - "get_model_by_country": "reads_models", - "list_model_versions": "reads_models", - "get_model_version": "reads_models", - "list_variables": "reads_variables", - "get_variable": "reads_variables", - "list_parameters": "reads_parameters", - "get_parameter": "reads_parameters", - "list_parameter_children": "reads_parameters", - "list_parameter_values": "reads_parameters", - "get_parameter_value": "reads_parameters", - "list_datasets": "reads_datasets", - "get_dataset": "reads_datasets", - "list_regions": "reads_regions", - "get_region": "reads_regions", - "get_region_by_code": "reads_regions", - "get_economy_options": "reads_regions", +def test_resource_entrypoints_are_defined_in_the_service_module() -> None: + method_names = { + "list_models", + "get_model", + "get_model_by_country", + "list_model_versions", + "get_model_version", + "list_variables", + "get_variable", + "list_parameters", + "get_parameter", + "list_parameter_children", + "list_parameter_values", + "get_parameter_value", + "list_datasets", + "get_dataset", + "list_regions", + "get_region", + "get_region_by_code", + "get_economy_options", } - for method_name, module_name in expected_modules.items(): + for method_name in method_names: method = getattr(V2MetadataService, method_name) - assert method.__module__.endswith(f".{module_name}") + assert method.__module__.endswith(".services") diff --git a/tests/unit/v2/test_policy_canonicalization.py b/tests/unit/v2/test_policy_canonicalization.py index bf92fb9da..3b750456b 100644 --- a/tests/unit/v2/test_policy_canonicalization.py +++ b/tests/unit/v2/test_policy_canonicalization.py @@ -6,13 +6,13 @@ import hashlib from uuid import UUID, uuid4 -from policyengine_api.services.v2.policies.canonicalization import ( +from policyengine_api.services.v2.policies.transformations import ( POLICY_CANONICALIZATION_VERSION, canonical_policy_document, canonicalize_policy, ) -from policyengine_api.services.v2.policies.commands import ( - ResolvedPolicyCreateCommand, +from policyengine_api.services.v2.policies.types import ( + ResolvedPolicyCreationInput, ) @@ -22,7 +22,7 @@ SECOND_PARAMETER_ID = UUID("00000000-0000-0000-0000-000000000040") -def _command(*, values=None, **changes) -> ResolvedPolicyCreateCommand: +def _command(*, values=None, **changes) -> ResolvedPolicyCreationInput: fields = { "country_id": "us", "tax_benefit_model_id": MODEL_ID, @@ -39,7 +39,7 @@ def _command(*, values=None, **changes) -> ResolvedPolicyCreateCommand: ], } fields.update(changes) - return ResolvedPolicyCreateCommand.model_validate(fields) + return ResolvedPolicyCreationInput.model_validate(fields) def test_document_has_versioned_deterministic_content_only() -> None: diff --git a/tests/unit/v2/test_policy_catalog.py b/tests/unit/v2/test_policy_catalog.py index c09a30b10..3133026dd 100644 --- a/tests/unit/v2/test_policy_catalog.py +++ b/tests/unit/v2/test_policy_catalog.py @@ -16,13 +16,13 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.services.v2.policies.catalog_validation import ( - PolicyCatalogValidationError, +from policyengine_api.services.v2.policies.services import ( + resolve_policy_creation_input, ) -from policyengine_api.services.v2.policies.creation import ( - resolve_policy_catalog, +from policyengine_api.services.v2.policies.types import PolicyCreationInput +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, ) -from policyengine_api.services.v2.policies.commands import PolicyCreateCommand def _catalog_session(): @@ -65,7 +65,7 @@ def _command(model_id, parameter_id=None, *, country_id="us"): "start_date": "2026-01-01T00:00:00Z", } ) - return PolicyCreateCommand( + return PolicyCreationInput( country_id=country_id, tax_benefit_model_id=model_id, parameter_values=parameter_values, @@ -75,7 +75,7 @@ def _command(model_id, parameter_id=None, *, country_id="us"): def test_resolver_binds_model_version_and_all_parameter_ids() -> None: engine, session, model, version, parameter, _previous = _catalog_session() try: - resolved = resolve_policy_catalog( + resolved = resolve_policy_creation_input( session, _command(model.id, parameter.id), policyengine_version="5.2.0", @@ -95,7 +95,7 @@ def test_resolver_binds_model_version_and_all_parameter_ids() -> None: def test_omitted_version_selects_the_running_catalog() -> None: engine, session, model, version, _parameter, _previous = _catalog_session() try: - resolved = resolve_policy_catalog( + resolved = resolve_policy_creation_input( session, _command(model.id), running_policyengine_version="5.2.0", @@ -110,7 +110,7 @@ def test_wrong_stable_model_is_rejected() -> None: engine, session, _model, _version, parameter, _previous = _catalog_session() try: with pytest.raises(PolicyCatalogValidationError, match="selected country"): - resolve_policy_catalog( + resolve_policy_creation_input( session, _command(uuid4(), parameter.id), policyengine_version="5.2.0", @@ -124,7 +124,7 @@ def test_parameter_from_another_model_version_is_rejected() -> None: engine, session, model, _version, _parameter, previous = _catalog_session() try: with pytest.raises(PolicyCatalogValidationError, match="every parameter_id"): - resolve_policy_catalog( + resolve_policy_creation_input( session, _command(model.id, previous.id), policyengine_version="5.2.0", @@ -138,13 +138,13 @@ def test_absent_or_unsupported_catalog_never_falls_back() -> None: engine, session, model, _version, _parameter, _previous = _catalog_session() try: with pytest.raises(MetadataCatalogVersionNotFoundError): - resolve_policy_catalog( + resolve_policy_creation_input( session, _command(model.id), policyengine_version="4.0.0", ) with pytest.raises(MetadataCatalogVersionNotFoundError): - resolve_policy_catalog( + resolve_policy_creation_input( session, _command(model.id, country_id="uk"), policyengine_version="5.2.0", diff --git a/tests/unit/v2/test_policy_commands.py b/tests/unit/v2/test_policy_inputs.py similarity index 73% rename from tests/unit/v2/test_policy_commands.py rename to tests/unit/v2/test_policy_inputs.py index 3502df5f9..9b3153604 100644 --- a/tests/unit/v2/test_policy_commands.py +++ b/tests/unit/v2/test_policy_inputs.py @@ -1,4 +1,4 @@ -"""Validation tests for immutable v2 policy commands.""" +"""Validation tests for immutable API v2 policy inputs.""" from __future__ import annotations @@ -9,12 +9,14 @@ from pydantic import ValidationError import pytest -from policyengine_api.services.v2.policies.commands import ( +from policyengine_api.services.v2.policies.types import ( + NativePolicyCreationInput, + PolicyCreationInput, + PolicyParameterValueInput, + ResolvedPolicyCreationInput, +) +from policyengine_api.services.v2.policies.validators import ( MAXIMUM_POLICY_PARAMETER_VALUES, - NativePolicyCreateCommand, - PolicyCreateCommand, - PolicyParameterValueCommand, - ResolvedPolicyCreateCommand, ) @@ -39,11 +41,11 @@ def _command(**changes) -> dict[str, object]: return fields -def test_command_normalizes_country_and_effective_dates_to_utc() -> None: - command = PolicyCreateCommand.model_validate(_command()) +def test_input_normalizes_country_and_effective_dates_to_utc() -> None: + policy_input = PolicyCreationInput.model_validate(_command()) - assert command.country_id == "us" - assert command.parameter_values[0].start_date == datetime( + assert policy_input.country_id == "us" + assert policy_input.parameter_values[0].start_date == datetime( 2026, 1, 1, @@ -51,11 +53,11 @@ def test_command_normalizes_country_and_effective_dates_to_utc() -> None: ) -def test_native_and_resolved_commands_keep_catalog_selection_explicit() -> None: - native = NativePolicyCreateCommand.model_validate( +def test_native_and_resolved_inputs_keep_catalog_selection_explicit() -> None: + native = NativePolicyCreationInput.model_validate( {**_command(), "policyengine_version": "5.2.0"} ) - resolved = ResolvedPolicyCreateCommand.model_validate( + resolved = ResolvedPolicyCreationInput.model_validate( { **native.model_dump(exclude={"policyengine_version"}), "policyengine_version": "5.2.0", @@ -81,7 +83,7 @@ def test_native_and_resolved_commands_keep_catalog_selection_explicit() -> None: ) def test_non_json_values_are_rejected(value: object) -> None: with pytest.raises(ValidationError): - PolicyParameterValueCommand.model_validate(_value(value=value)) + PolicyParameterValueInput.model_validate(_value(value=value)) def test_json_reference_cycles_are_rejected() -> None: @@ -89,7 +91,7 @@ def test_json_reference_cycles_are_rejected() -> None: cyclic.append(cyclic) with pytest.raises(ValidationError, match="reference cycles"): - PolicyParameterValueCommand.model_validate(_value(value=cyclic)) + PolicyParameterValueInput.model_validate(_value(value=cyclic)) @pytest.mark.parametrize( @@ -104,7 +106,7 @@ def test_json_reference_cycles_are_rejected() -> None: ) def test_invalid_effective_dates_are_rejected(changes: dict[str, object]) -> None: with pytest.raises(ValidationError): - PolicyParameterValueCommand.model_validate(_value(**changes)) + PolicyParameterValueInput.model_validate(_value(**changes)) def test_duplicate_parameter_and_normalized_start_date_is_rejected() -> None: @@ -119,14 +121,14 @@ def test_duplicate_parameter_and_normalized_start_date_is_rejected() -> None: ) with pytest.raises(ValidationError, match="parameter_id/start_date"): - PolicyCreateCommand.model_validate( + PolicyCreationInput.model_validate( _command(parameter_values=[first, duplicate]) ) def test_parameter_value_count_is_bounded_but_empty_policy_is_valid() -> None: assert ( - PolicyCreateCommand.model_validate( + PolicyCreationInput.model_validate( _command(parameter_values=[]) ).parameter_values == [] @@ -143,4 +145,4 @@ def test_parameter_value_count_is_bounded_but_empty_policy_is_valid() -> None: for index in range(MAXIMUM_POLICY_PARAMETER_VALUES + 1) ] with pytest.raises(ValidationError): - PolicyCreateCommand.model_validate(_command(parameter_values=parameter_values)) + PolicyCreationInput.model_validate(_command(parameter_values=parameter_values)) diff --git a/tests/unit/v2/test_policy_legacy_translation.py b/tests/unit/v2/test_policy_legacy_translation.py index 2f29f348e..bf569616e 100644 --- a/tests/unit/v2/test_policy_legacy_translation.py +++ b/tests/unit/v2/test_policy_legacy_translation.py @@ -14,12 +14,18 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.services.v2.policies.legacy_translation import ( - LegacyPolicySnapshot, - LegacyPolicyTranslationError, +from policyengine_api.services.v2.policies.database_connectors.reads import ( + read_parameters_by_name, + read_policy_catalog, +) +from policyengine_api.services.v2.policies.transformations import ( parse_legacy_period, translate_legacy_policy, ) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyTranslationError, +) def _session_and_catalog(): @@ -62,6 +68,31 @@ def _snapshot(**changes) -> LegacyPolicySnapshot: return LegacyPolicySnapshot.model_validate(fields) +def _translate( + session: Session, + snapshot: LegacyPolicySnapshot, + *, + running_policyengine_version: str = "5.2.0", +): + selected = read_policy_catalog( + session, + snapshot.country_id, + running_policyengine_version=running_policyengine_version, + ) + assert isinstance(snapshot.policy_json, dict) + parameters = read_parameters_by_name( + session, + model_version_id=selected.model_version.id, + names=set(snapshot.policy_json), + ) + return translate_legacy_policy( + snapshot, + selected=selected, + parameters=parameters, + country_package_versions={"us": "1.0.0"}, + ) + + def test_year_day_and_explicit_range_periods_are_inclusive_utc() -> None: assert parse_legacy_period("2026") == ( datetime(2026, 1, 1, tzinfo=timezone.utc), @@ -80,12 +111,7 @@ def test_year_day_and_explicit_range_periods_are_inclusive_utc() -> None: def test_translation_resolves_paths_and_excludes_legacy_identity_and_label() -> None: engine, session, model, version, first, second = _session_and_catalog() try: - translated = translate_legacy_policy( - session, - _snapshot(), - running_policyengine_version="5.2.0", - country_package_versions={"us": "1.0.0"}, - ) + translated = _translate(session, _snapshot()) assert translated.tax_benefit_model_id == model.id assert translated.tax_benefit_model_version_id == version.id @@ -103,18 +129,8 @@ def test_translation_resolves_paths_and_excludes_legacy_identity_and_label() -> def test_label_does_not_change_translated_core_content() -> None: engine, session, _model, _version, _first, _second = _session_and_catalog() try: - first = translate_legacy_policy( - session, - _snapshot(label="First"), - running_policyengine_version="5.2.0", - country_package_versions={"us": "1.0.0"}, - ) - second = translate_legacy_policy( - session, - _snapshot(label="Second", legacy_policy_id=43), - running_policyengine_version="5.2.0", - country_package_versions={"us": "1.0.0"}, - ) + first = _translate(session, _snapshot(label="First")) + second = _translate(session, _snapshot(label="Second", legacy_policy_id=43)) assert first == second finally: session.close() @@ -141,12 +157,7 @@ def test_missing_paths_malformed_periods_and_conflicts_fail( engine, session, _model, _version, _first, _second = _session_and_catalog() try: with pytest.raises(LegacyPolicyTranslationError): - translate_legacy_policy( - session, - _snapshot(policy_json=policy_json), - running_policyengine_version="5.2.0", - country_package_versions={"us": "1.0.0"}, - ) + _translate(session, _snapshot(policy_json=policy_json)) finally: session.close() engine.dispose() @@ -156,12 +167,7 @@ def test_country_package_version_must_match_running_release() -> None: engine, session, _model, _version, _first, _second = _session_and_catalog() try: with pytest.raises(LegacyPolicyTranslationError, match="api_version"): - translate_legacy_policy( - session, - _snapshot(api_version="0.9.0"), - running_policyengine_version="5.2.0", - country_package_versions={"us": "1.0.0"}, - ) + _translate(session, _snapshot(api_version="0.9.0")) finally: session.close() engine.dispose() @@ -190,11 +196,10 @@ def test_unknown_policyengine_version_never_falls_back() -> None: engine, session, _model, _version, _first, _second = _session_and_catalog() try: with pytest.raises(Exception, match="running PolicyEngine.py"): - translate_legacy_policy( + _translate( session, _snapshot(), running_policyengine_version="4.0.0", - country_package_versions={"us": "1.0.0"}, ) finally: session.close() diff --git a/tests/unit/v2/test_policy_persistence_statements.py b/tests/unit/v2/test_policy_persistence_statements.py index b9ac908ec..6e03a11cf 100644 --- a/tests/unit/v2/test_policy_persistence_statements.py +++ b/tests/unit/v2/test_policy_persistence_statements.py @@ -4,7 +4,7 @@ from sqlalchemy.dialects import postgresql -from policyengine_api.data.v2.policies import creates +from policyengine_api.services.v2.policies.database_connectors import creates def test_policy_insert_uses_the_content_identity_constraint_and_returning() -> None: diff --git a/tests/unit/v2/test_policy_query.py b/tests/unit/v2/test_policy_query.py index 96fe15fe4..ebc9748cd 100644 --- a/tests/unit/v2/test_policy_query.py +++ b/tests/unit/v2/test_policy_query.py @@ -16,10 +16,12 @@ TaxBenefitModelVersion, V2_METADATA, ) -from policyengine_api.data.v2.policies.reads import ( +from policyengine_api.services.v2.policies.services import ( + read_complete_policy, + read_policy_page, +) +from policyengine_api.services.v2.policies.validators import ( PolicyNotFoundError, - list_policies, - read_policy, ) @@ -102,7 +104,7 @@ def _stored_policies(): def test_detail_joins_parameter_names_and_orders_complete_values() -> None: engine, session, _model, first, _second, _other = _stored_policies() try: - result = read_policy(session, country_id="us", policy_id=first.id) + result = read_complete_policy(session, country_id="us", policy_id=first.id) assert result.id == first.id assert result.created_at == first.created_at @@ -121,7 +123,7 @@ def test_detail_uses_country_as_part_of_resource_identity() -> None: engine, session, _model, first, _second, _other = _stored_policies() try: with pytest.raises(PolicyNotFoundError): - read_policy(session, country_id="uk", policy_id=first.id) + read_complete_policy(session, country_id="uk", policy_id=first.id) finally: session.close() engine.dispose() @@ -131,7 +133,7 @@ def test_empty_policy_has_an_empty_nested_collection() -> None: engine, session, _model, _first, second, _other = _stored_policies() try: assert ( - read_policy( + read_complete_policy( session, country_id="us", policy_id=second.id, @@ -146,14 +148,14 @@ def test_empty_policy_has_an_empty_nested_collection() -> None: def test_collection_filters_orders_paginates_and_returns_complete_items() -> None: engine, session, model, first, second, _other = _stored_policies() try: - first_page = list_policies( + first_page = read_policy_page( session, country_id="us", tax_benefit_model_id=model.id, offset=0, limit=1, ) - second_page = list_policies( + second_page = read_policy_page( session, country_id="us", tax_benefit_model_id=model.id, @@ -176,10 +178,12 @@ def test_collection_filters_orders_paginates_and_returns_complete_items() -> Non def test_model_filter_is_exact() -> None: engine, session, _model, _first, _second, _other = _stored_policies() try: - result = list_policies( + result = read_policy_page( session, country_id="us", tax_benefit_model_id=UUID("ffffffff-ffff-ffff-ffff-ffffffffffff"), + offset=0, + limit=100, ) assert result.items == () assert result.has_more is False diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py index 254674420..ebbe81863 100644 --- a/tests/unit/v2/test_policy_routes.py +++ b/tests/unit/v2/test_policy_routes.py @@ -15,18 +15,17 @@ MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, ) -from policyengine_api.data.v2.policies.reads import ( - PolicyNotFoundError, +from policyengine_api.services.v2.policies.types import ( + NativePolicyCreation, PolicyPage, PolicyParameterValueRead, PolicyRead, ) -from policyengine_api.services.v2.policies.catalog_validation import ( +from policyengine_api.services.v2.policies.validators import ( PolicyCatalogValidationError, -) -from policyengine_api.services.v2.policies.creation import ( PolicyContentHashCollisionError, PolicyCreationIntegrityError, + PolicyNotFoundError, ) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -34,7 +33,6 @@ RouteImplementation, RouteImplementationSettings, ) -from policyengine_api.services.v2.policies.service import NativePolicyCreation POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") From d5e4bec15a9e43badacf4d856fe0af5f17b12fe5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:41:40 +0400 Subject: [PATCH 12/18] Refactor v2 user-policy layers --- .../skills/v2-code-organization.md | 5 +- .../data/v2/user_policies/__init__.py | 1 - .../data/v2/user_policies/reads.py | 181 -------- .../fastapi_routes/dependencies.py | 21 +- .../v2/user_policies/request_models.py | 10 +- .../v2/user_policies/response_models.py | 2 +- .../fastapi_routes/v2/user_policies/routes.py | 6 +- .../services/user_policy_mirroring.py | 17 +- .../services/user_policy_service.py | 6 +- .../services/v2/user_policies/commands.py | 34 -- .../database_connectors/__init__.py | 1 + .../database_connectors}/creates.py | 30 +- .../database_connectors}/deletes.py | 6 +- .../database_connectors/reads.py | 96 +++++ .../database_connectors}/updates.py | 12 +- .../v2/user_policies/database_session.py | 26 ++ .../v2/user_policies/legacy_service.py | 285 ------------- .../v2/user_policies/legacy_translation.py | 74 ---- .../services/v2/user_policies/service.py | 159 ------- .../services/v2/user_policies/services.py | 394 ++++++++++++++++++ .../v2/user_policies/transformations.py | 85 ++++ .../services/v2/user_policies/types.py | 99 +++++ .../services/v2/user_policies/validators.py | 155 +++++++ pyproject.toml | 1 - .../contract/test_policy_v2_compatibility.py | 4 +- tests/contract/test_v1_route_contracts.py | 4 +- .../test_v1_user_policy_dual_write.py | 11 +- .../test_v2_user_policy_mirroring.py | 30 +- .../test_user_policy_dual_write_routes.py | 4 +- .../services/test_user_policy_mirroring.py | 10 +- .../unit/services/test_user_policy_service.py | 2 +- tests/unit/v2/test_data_crud_boundaries.py | 34 +- tests/unit/v2/test_user_policy_legacy.py | 10 +- tests/unit/v2/test_user_policy_routes.py | 22 +- tests/unit/v2/test_user_policy_service.py | 31 +- 35 files changed, 982 insertions(+), 886 deletions(-) delete mode 100644 policyengine_api/data/v2/user_policies/__init__.py delete mode 100644 policyengine_api/data/v2/user_policies/reads.py delete mode 100644 policyengine_api/services/v2/user_policies/commands.py create mode 100644 policyengine_api/services/v2/user_policies/database_connectors/__init__.py rename policyengine_api/{data/v2/user_policies => services/v2/user_policies/database_connectors}/creates.py (69%) rename policyengine_api/{data/v2/user_policies => services/v2/user_policies/database_connectors}/deletes.py (63%) create mode 100644 policyengine_api/services/v2/user_policies/database_connectors/reads.py rename policyengine_api/{data/v2/user_policies => services/v2/user_policies/database_connectors}/updates.py (71%) create mode 100644 policyengine_api/services/v2/user_policies/database_session.py delete mode 100644 policyengine_api/services/v2/user_policies/legacy_service.py delete mode 100644 policyengine_api/services/v2/user_policies/legacy_translation.py delete mode 100644 policyengine_api/services/v2/user_policies/service.py create mode 100644 policyengine_api/services/v2/user_policies/services.py create mode 100644 policyengine_api/services/v2/user_policies/transformations.py create mode 100644 policyengine_api/services/v2/user_policies/types.py create mode 100644 policyengine_api/services/v2/user_policies/validators.py diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index 330b6ec76..dd29a98ef 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -52,9 +52,8 @@ specific descriptor when one operation file would become too broad, such as Policies currently have `creates.py` and `reads.py` because policies are immutable. Metadata currently has only read connector modules because metadata -routes are read-only. User-policy associations support creation, reading, -updating, and deletion; migrate that existing package to this layout when its -modules are next moved or substantially changed. +routes are read-only. User-policy associations have `creates.py`, `reads.py`, +`updates.py`, and `deletes.py` because they support all four operations. ### `services.py` diff --git a/policyengine_api/data/v2/user_policies/__init__.py b/policyengine_api/data/v2/user_policies/__init__.py deleted file mode 100644 index 375e2b4dd..000000000 --- a/policyengine_api/data/v2/user_policies/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Database CRUD operations for v2 user-policy associations.""" diff --git a/policyengine_api/data/v2/user_policies/reads.py b/policyengine_api/data/v2/user_policies/reads.py deleted file mode 100644 index e6940d77e..000000000 --- a/policyengine_api/data/v2/user_policies/reads.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Database reads used by v2 user-policy association operations.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime -from uuid import UUID - -from sqlmodel import Session, col, select - -from policyengine_api.data.v2.models import ( - LegacyUserMapping, - LegacyUserPolicyMapping, - Policy, - User, - UserPolicy, -) - - -class UserPolicyNotFoundError(LookupError): - """Raised when an association is absent from the selected country.""" - - -@dataclass(frozen=True) -class UserPolicyRead: - id: UUID - country_id: str - user_id: UUID - policy_id: UUID - name: str | None - description: str | None - created_at: datetime - updated_at: datetime - - -@dataclass(frozen=True) -class UserPolicyPage: - items: tuple[UserPolicyRead, ...] - offset: int - limit: int - has_more: bool - - -def read_user(session: Session, user_id: UUID) -> User | None: - """Read one v2 user by UUID.""" - - return session.get(User, user_id) - - -def read_policy_for_association(session: Session, policy_id: UUID) -> Policy | None: - """Read one policy referenced by an association create command.""" - - return session.exec(select(Policy).where(Policy.id == policy_id)).one_or_none() - - -def read_legacy_user_mapping( - session: Session, - legacy_user_id: str, - *, - lock: bool, -) -> LegacyUserMapping | None: - """Read one legacy-user-to-v2-user mapping.""" - - statement = select(LegacyUserMapping).where( - LegacyUserMapping.legacy_user_id == legacy_user_id - ) - if lock: - statement = statement.with_for_update() - return session.exec(statement).one_or_none() - - -def read_legacy_user_policy_mapping( - session: Session, - *, - country_id: str, - legacy_user_policy_id: int, - lock: bool, -) -> LegacyUserPolicyMapping | None: - """Read one country-scoped legacy association mapping.""" - - statement = select(LegacyUserPolicyMapping).where( - LegacyUserPolicyMapping.country_id == country_id, - LegacyUserPolicyMapping.legacy_user_policy_id == legacy_user_policy_id, - ) - if lock: - statement = statement.with_for_update() - return session.exec(statement).one_or_none() - - -def read_mapped_user_policy( - session: Session, - mapping: LegacyUserPolicyMapping, -) -> UserPolicy | None: - """Read the association referenced by one legacy mapping.""" - - return session.exec( - select(UserPolicy).where( - UserPolicy.id == mapping.user_policy_id, - UserPolicy.country_id == mapping.country_id, - ) - ).one_or_none() - - -def association_read(association: UserPolicy) -> UserPolicyRead: - return UserPolicyRead( - id=association.id, - country_id=association.country_id, - user_id=association.user_id, - policy_id=association.policy_id, - name=association.name, - description=association.description, - created_at=association.created_at, - updated_at=association.updated_at, - ) - - -def get_user_policy_row( - session: Session, - *, - country_id: str, - association_id: UUID, -) -> UserPolicy: - association = session.exec( - select(UserPolicy).where( - UserPolicy.id == association_id, - UserPolicy.country_id == country_id, - ) - ).one_or_none() - if association is None: - raise UserPolicyNotFoundError( - f"user-policy association {association_id} was not found" - ) - return association - - -def read_user_policy( - session: Session, - *, - country_id: str, - association_id: UUID, -) -> UserPolicyRead: - """Read one association only under its stored country.""" - - return association_read( - get_user_policy_row( - session, - country_id=country_id, - association_id=association_id, - ) - ) - - -def list_user_policies( - session: Session, - *, - country_id: str, - user_id: UUID, - policy_id: UUID | None = None, - offset: int = 0, - limit: int = 100, -) -> UserPolicyPage: - """Read one deterministic bounded page for a v2 user UUID.""" - - statement = select(UserPolicy).where( - UserPolicy.country_id == country_id, - UserPolicy.user_id == user_id, - ) - if policy_id is not None: - statement = statement.where(UserPolicy.policy_id == policy_id) - rows = session.exec( - statement.order_by(col(UserPolicy.created_at), col(UserPolicy.id)) - .offset(offset) - .limit(limit + 1) - ).all() - has_more = len(rows) > limit - return UserPolicyPage( - items=tuple(association_read(row) for row in rows[:limit]), - offset=offset, - limit=limit, - has_more=has_more, - ) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 058d8eac4..9352a7074 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -33,13 +33,11 @@ PolicyPage, PolicyRead, ) - from policyengine_api.data.v2.user_policies.reads import ( + from policyengine_api.services.v2.user_policies.types import ( + UserPolicyCreationInput, UserPolicyPage, UserPolicyRead, - ) - from policyengine_api.services.v2.user_policies.commands import ( - UserPolicyCreateCommand, - UserPolicyPatchCommand, + UserPolicyUpdateInput, ) @@ -227,7 +225,7 @@ class V2UserPolicyResourceService(Protocol): def create_user_policy( self, - command: "UserPolicyCreateCommand", + association_input: "UserPolicyCreationInput", ) -> "UserPolicyRead": ... def get_user_policy( @@ -252,7 +250,7 @@ def patch_user_policy( *, country_id: str, association_id: UUID, - command: "UserPolicyPatchCommand", + association_input: "UserPolicyUpdateInput", ) -> "UserPolicyRead": ... def delete_user_policy( @@ -326,9 +324,14 @@ def _default_v2_policy_service_factory() -> V2PolicyResourceService: def _default_v2_user_policy_service_factory() -> V2UserPolicyResourceService: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.user_policies.service import V2UserPolicyService + from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, + ) + from policyengine_api.services.v2.user_policies.services import ( + V2UserPolicyService, + ) - return V2UserPolicyService(get_v2_session_factory()) + return V2UserPolicyService(UserPolicyDatabaseSession(get_v2_session_factory())) @dataclass(frozen=True) diff --git a/policyengine_api/fastapi_routes/v2/user_policies/request_models.py b/policyengine_api/fastapi_routes/v2/user_policies/request_models.py index e568d3d4b..9292bc440 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/request_models.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/request_models.py @@ -1,14 +1,14 @@ """Strict HTTP request models for native v2 user-policy associations.""" -from policyengine_api.services.v2.user_policies.commands import ( - UserPolicyCreateCommand, - UserPolicyPatchCommand, +from policyengine_api.services.v2.user_policies.types import ( + UserPolicyCreationInput, + UserPolicyUpdateInput, ) -class UserPolicyCreateRequest(UserPolicyCreateCommand): +class UserPolicyCreateRequest(UserPolicyCreationInput): """Association identity, immutable link fields, and presentation fields.""" -class UserPolicyPatchRequest(UserPolicyPatchCommand): +class UserPolicyPatchRequest(UserPolicyUpdateInput): """Explicitly supplied mutable presentation fields.""" diff --git a/policyengine_api/fastapi_routes/v2/user_policies/response_models.py b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py index e01f167a9..110a18e39 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, StringConstraints -from policyengine_api.data.v2.user_policies.reads import ( +from policyengine_api.services.v2.user_policies.types import ( UserPolicyPage, UserPolicyRead, ) diff --git a/policyengine_api/fastapi_routes/v2/user_policies/routes.py b/policyengine_api/fastapi_routes/v2/user_policies/routes.py index 6bdd84524..620ee2703 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/routes.py @@ -24,10 +24,8 @@ UserPolicyPageResponse, UserPolicyPageResult, ) -from policyengine_api.data.v2.user_policies.reads import ( +from policyengine_api.services.v2.user_policies.validators import ( UserPolicyNotFoundError, -) -from policyengine_api.services.v2.user_policies.service import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, @@ -186,7 +184,7 @@ def patch() -> UserPolicyDetailResponse: item = service_factory().patch_user_policy( country_id=query.country_id, association_id=association_id, - command=body, + association_input=body, ) return UserPolicyDetailResponse( result=UserPolicyDetailResult(item=UserPolicyItem.from_read(item)) diff --git a/policyengine_api/services/user_policy_mirroring.py b/policyengine_api/services/user_policy_mirroring.py index 7da0742fd..5cf316ace 100644 --- a/policyengine_api/services/user_policy_mirroring.py +++ b/policyengine_api/services/user_policy_mirroring.py @@ -20,14 +20,14 @@ PolicyContentHashCollisionError, PolicyCreationIntegrityError, ) -from policyengine_api.services.v2.user_policies.legacy_service import ( - LegacyUserPolicyIntegrityError, +from policyengine_api.services.v2.user_policies.types import ( LegacyUserPolicyPersistenceResult, -) -from policyengine_api.data.v2.settings import V2ConfigurationError -from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, ) +from policyengine_api.services.v2.user_policies.validators import ( + LegacyUserPolicyIntegrityError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.gcp_logging import logger from policyengine_api.services.policy_service import PolicyService from policyengine_api.services.user_policy_service import ( @@ -53,11 +53,14 @@ class UserPolicyMirrorUnavailableError(RuntimeError): def _default_mirror_factory() -> LegacyUserPolicyMirror: from policyengine_api.data.v2.database import get_v2_session_factory - from policyengine_api.services.v2.user_policies.service import ( + from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, + ) + from policyengine_api.services.v2.user_policies.services import ( V2UserPolicyService, ) - return V2UserPolicyService(get_v2_session_factory()) + return V2UserPolicyService(UserPolicyDatabaseSession(get_v2_session_factory())) def _failure_category(error: Exception) -> str: diff --git a/policyengine_api/services/user_policy_service.py b/policyengine_api/services/user_policy_service.py index 15035978c..60904dd85 100644 --- a/policyengine_api/services/user_policy_service.py +++ b/policyengine_api/services/user_policy_service.py @@ -17,11 +17,11 @@ from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent -from policyengine_api.services.v2.user_policies.legacy_service import ( +from policyengine_api.services.v2.user_policies.types import ( LegacyUserPolicyPersistenceResult, -) -from policyengine_api.services.v2.user_policies.legacy_translation import ( LegacyUserPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.transformations import ( fingerprint_legacy_user_policy, ) diff --git a/policyengine_api/services/v2/user_policies/commands.py b/policyengine_api/services/v2/user_policies/commands.py deleted file mode 100644 index 933f0d2bd..000000000 --- a/policyengine_api/services/v2/user_policies/commands.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Strict application commands for user-policy associations.""" - -from __future__ import annotations - -from typing import Annotated - -from pydantic import BaseModel, ConfigDict, StringConstraints, model_validator - -from policyengine_api.query_parameters import CountryId, ResourceId, UserId - - -class StrictAssociationCommand(BaseModel): - """Reject fields outside the reviewed association contract.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - -class UserPolicyCreateCommand(StrictAssociationCommand): - country_id: CountryId - user_id: UserId - policy_id: ResourceId - name: Annotated[str, StringConstraints(max_length=255)] | None = None - description: str | None = None - - -class UserPolicyPatchCommand(StrictAssociationCommand): - name: Annotated[str, StringConstraints(max_length=255)] | None = None - description: str | None = None - - @model_validator(mode="after") - def require_supplied_field(self) -> "UserPolicyPatchCommand": - if not self.model_fields_set.intersection({"name", "description"}): - raise ValueError("At least one of name or description must be supplied") - return self diff --git a/policyengine_api/services/v2/user_policies/database_connectors/__init__.py b/policyengine_api/services/v2/user_policies/database_connectors/__init__.py new file mode 100644 index 000000000..47b6f091b --- /dev/null +++ b/policyengine_api/services/v2/user_policies/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for v2 user-policy services.""" diff --git a/policyengine_api/data/v2/user_policies/creates.py b/policyengine_api/services/v2/user_policies/database_connectors/creates.py similarity index 69% rename from policyengine_api/data/v2/user_policies/creates.py rename to policyengine_api/services/v2/user_policies/database_connectors/creates.py index c8e7b6904..e8d48701f 100644 --- a/policyengine_api/data/v2/user_policies/creates.py +++ b/policyengine_api/services/v2/user_policies/database_connectors/creates.py @@ -1,4 +1,4 @@ -"""Database creates used by v2 user-policy association operations.""" +"""Database inserts used by v2 user-policy operations.""" from __future__ import annotations @@ -13,31 +13,20 @@ User, UserPolicy, ) -from policyengine_api.services.v2.user_policies.commands import ( - UserPolicyCreateCommand, -) +from policyengine_api.services.v2.user_policies.types import UserPolicyCreationInput def create_user_policy( - session: Session, - command: UserPolicyCreateCommand, + session: Session, association_input: UserPolicyCreationInput ) -> UserPolicy: - """Create one independently identified association.""" - - association = UserPolicy(**command.model_dump()) + association = UserPolicy(**association_input.model_dump()) session.add(association) session.flush() session.refresh(association) return association -def create_transition_user( - session: Session, - *, - primary_country: str, -) -> User: - """Create a minimal v2 user for one legacy identity.""" - +def create_transition_user(session: Session, *, primary_country: str) -> User: user = User(primary_country=primary_country) session.add(user) session.flush() @@ -45,13 +34,8 @@ def create_transition_user( def create_legacy_user_mapping( - session: Session, - *, - legacy_user_id: str, - user_id: UUID, + session: Session, *, legacy_user_id: str, user_id: UUID ) -> UUID | None: - """Create one legacy-user mapping, or return none after a conflict.""" - return session.execute( insert(LegacyUserMapping) .values(legacy_user_id=legacy_user_id, user_id=user_id) @@ -70,8 +54,6 @@ def create_legacy_user_policy_mapping( fingerprint_version: int, fingerprint: str, ) -> UUID | None: - """Create one legacy association mapping, or return none after a conflict.""" - return session.execute( insert(LegacyUserPolicyMapping) .values( diff --git a/policyengine_api/data/v2/user_policies/deletes.py b/policyengine_api/services/v2/user_policies/database_connectors/deletes.py similarity index 63% rename from policyengine_api/data/v2/user_policies/deletes.py rename to policyengine_api/services/v2/user_policies/database_connectors/deletes.py index 70594aeca..7b0a077de 100644 --- a/policyengine_api/data/v2/user_policies/deletes.py +++ b/policyengine_api/services/v2/user_policies/database_connectors/deletes.py @@ -1,4 +1,4 @@ -"""Database deletes used by v2 user-policy association operations.""" +"""Database deletes used by v2 user-policy operations.""" from __future__ import annotations @@ -8,14 +8,10 @@ def delete_user_policy(session: Session, association: UserPolicy) -> None: - """Delete one association and flush its database cascades.""" - session.delete(association) session.flush() def delete_transition_user(session: Session, user: User) -> None: - """Delete an unreferenced transition user after a mapping conflict.""" - session.delete(user) session.flush() diff --git a/policyengine_api/services/v2/user_policies/database_connectors/reads.py b/policyengine_api/services/v2/user_policies/database_connectors/reads.py new file mode 100644 index 000000000..66a12624f --- /dev/null +++ b/policyengine_api/services/v2/user_policies/database_connectors/reads.py @@ -0,0 +1,96 @@ +"""Database selections used by v2 user-policy operations.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import ( + LegacyUserMapping, + LegacyUserPolicyMapping, + Policy, + User, + UserPolicy, +) + + +def read_user(session: Session, user_id: UUID) -> User | None: + return session.get(User, user_id) + + +def read_policy_for_association(session: Session, policy_id: UUID) -> Policy | None: + return session.exec(select(Policy).where(Policy.id == policy_id)).one_or_none() + + +def read_legacy_user_mapping( + session: Session, legacy_user_id: str, *, lock: bool +) -> LegacyUserMapping | None: + statement = select(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == legacy_user_id + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def read_legacy_user_policy_mapping( + session: Session, + *, + country_id: str, + legacy_user_policy_id: int, + lock: bool, +) -> LegacyUserPolicyMapping | None: + statement = select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.country_id == country_id, + LegacyUserPolicyMapping.legacy_user_policy_id == legacy_user_policy_id, + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def read_mapped_user_policy( + session: Session, mapping: LegacyUserPolicyMapping +) -> UserPolicy | None: + return session.exec( + select(UserPolicy).where( + UserPolicy.id == mapping.user_policy_id, + UserPolicy.country_id == mapping.country_id, + ) + ).one_or_none() + + +def read_user_policy_row( + session: Session, *, country_id: str, association_id: UUID +) -> UserPolicy | None: + return session.exec( + select(UserPolicy).where( + UserPolicy.id == association_id, + UserPolicy.country_id == country_id, + ) + ).one_or_none() + + +def read_user_policy_rows( + session: Session, + *, + country_id: str, + user_id: UUID, + policy_id: UUID | None, + offset: int, + limit: int, +) -> list[UserPolicy]: + statement = select(UserPolicy).where( + UserPolicy.country_id == country_id, + UserPolicy.user_id == user_id, + ) + if policy_id is not None: + statement = statement.where(UserPolicy.policy_id == policy_id) + return list( + session.exec( + statement.order_by(col(UserPolicy.created_at), col(UserPolicy.id)) + .offset(offset) + .limit(limit + 1) + ).all() + ) diff --git a/policyengine_api/data/v2/user_policies/updates.py b/policyengine_api/services/v2/user_policies/database_connectors/updates.py similarity index 71% rename from policyengine_api/data/v2/user_policies/updates.py rename to policyengine_api/services/v2/user_policies/database_connectors/updates.py index 393bc730d..c58fd4d12 100644 --- a/policyengine_api/data/v2/user_policies/updates.py +++ b/policyengine_api/services/v2/user_policies/database_connectors/updates.py @@ -1,4 +1,4 @@ -"""Database updates used by v2 user-policy association operations.""" +"""Database updates used by v2 user-policy operations.""" from __future__ import annotations @@ -6,17 +6,15 @@ from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy from policyengine_api.data.v2.models.base import utc_now -from policyengine_api.services.v2.user_policies.commands import UserPolicyPatchCommand +from policyengine_api.services.v2.user_policies.types import UserPolicyUpdateInput def update_user_policy( session: Session, association: UserPolicy, - command: UserPolicyPatchCommand, + association_input: UserPolicyUpdateInput, ) -> UserPolicy: - """Update only explicitly supplied association presentation fields.""" - - for field_name, value in command.model_dump(exclude_unset=True).items(): + for field_name, value in association_input.model_dump(exclude_unset=True).items(): setattr(association, field_name, value) association.updated_at = utc_now() session.add(association) @@ -35,8 +33,6 @@ def update_legacy_user_policy_state( fingerprint: str, source_revision: int, ) -> None: - """Update an association projection and its applied source revision.""" - if update_name: association.name = reform_label association.updated_at = utc_now() diff --git a/policyengine_api/services/v2/user_policies/database_session.py b/policyengine_api/services/v2/user_policies/database_session.py new file mode 100644 index 000000000..f7576c23b --- /dev/null +++ b/policyengine_api/services/v2/user_policies/database_session.py @@ -0,0 +1,26 @@ +"""Database session lifetime management for v2 user-policy services.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + + +class UserPolicyDatabaseSession: + """Open read sessions and transactions for user-policy services.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + with self._session_factory() as session: + yield session + + @contextmanager + def transaction(self) -> Iterator[Session]: + with self._session_factory.begin() as session: + yield session diff --git a/policyengine_api/services/v2/user_policies/legacy_service.py b/policyengine_api/services/v2/user_policies/legacy_service.py deleted file mode 100644 index 50dec4913..000000000 --- a/policyengine_api/services/v2/user_policies/legacy_service.py +++ /dev/null @@ -1,285 +0,0 @@ -"""Transactional operations for mirroring v1 saved policies into v2.""" - -from __future__ import annotations - -from dataclasses import dataclass -from uuid import UUID - -from sqlmodel import Session - -from policyengine_api.data.v2.models import LegacyUserPolicyMapping -from policyengine_api.data.v2.user_policies.creates import ( - create_legacy_user_mapping, - create_legacy_user_policy_mapping, - create_transition_user, - create_user_policy, -) -from policyengine_api.data.v2.user_policies.deletes import ( - delete_transition_user, - delete_user_policy, -) -from policyengine_api.data.v2.user_policies.reads import ( - read_legacy_user_mapping, - read_legacy_user_policy_mapping, - read_mapped_user_policy, - read_user, -) -from policyengine_api.data.v2.user_policies.updates import ( - update_legacy_user_policy_state, -) -from policyengine_api.services.v2.policies.services import ( - mirror_legacy_policy_in_session, -) -from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot -from policyengine_api.services.v2.user_policies.commands import ( - UserPolicyCreateCommand, -) -from policyengine_api.services.v2.user_policies.legacy_translation import ( - USER_POLICY_FINGERPRINT_VERSION, - LegacyUserPolicySnapshot, - fingerprint_legacy_user_policy, - project_legacy_user_policy, -) - - -class LegacyUserPolicyIntegrityError(RuntimeError): - """Raised when source, policy, association, or mapping identity conflicts.""" - - -@dataclass(frozen=True) -class LegacyUserPolicyPersistenceResult: - association_id: UUID - policy_id: UUID - association_created: bool - association_updated: bool - mapping_created: bool - - -def resolve_legacy_user_id( - session: Session, - *, - legacy_user_id: str, - primary_country: str, -) -> UUID: - """Return one durable v2 UUID for an exact opaque v1 user identifier.""" - - existing = read_legacy_user_mapping(session, legacy_user_id, lock=True) - if existing is not None: - if read_user(session, existing.user_id) is None: - raise LegacyUserPolicyIntegrityError( - "legacy user mapping has no referenced v2 user" - ) - return existing.user_id - - user = create_transition_user(session, primary_country=primary_country) - created_user_id = create_legacy_user_mapping( - session, - legacy_user_id=legacy_user_id, - user_id=user.id, - ) - if created_user_id is not None: - return created_user_id - - delete_transition_user(session, user) - concurrent = read_legacy_user_mapping(session, legacy_user_id, lock=False) - if concurrent is None or read_user(session, concurrent.user_id) is None: - raise LegacyUserPolicyIntegrityError( - "legacy user mapping conflict did not resolve to a v2 user" - ) - return concurrent.user_id - - -def apply_existing_legacy_user_policy_mapping( - session: Session, - *, - mapping: LegacyUserPolicyMapping, - country_id: str, - reform_label: str | None, - fingerprint: str, - user_id: UUID, - policy_id: UUID, - changed_fields: frozenset[str], - source_revision: int, -) -> LegacyUserPolicyPersistenceResult: - """Validate and, when newer, update one existing association mapping.""" - - association = read_mapped_user_policy(session, mapping) - if association is None: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy mapping has no association" - ) - if ( - association.policy_id != policy_id - or association.country_id != country_id - or association.user_id != user_id - ): - raise LegacyUserPolicyIntegrityError( - "legacy user-policy mapping conflicts with immutable association fields" - ) - if mapping.fingerprint_version != USER_POLICY_FINGERPRINT_VERSION: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy fingerprint version is unsupported" - ) - if source_revision < mapping.last_applied_source_revision: - return LegacyUserPolicyPersistenceResult( - association_id=association.id, - policy_id=policy_id, - association_created=False, - association_updated=False, - mapping_created=False, - ) - if source_revision == mapping.last_applied_source_revision: - if mapping.fingerprint_sha256 != fingerprint: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy revision conflicts with its stored fingerprint" - ) - return LegacyUserPolicyPersistenceResult( - association_id=association.id, - policy_id=policy_id, - association_created=False, - association_updated=False, - mapping_created=False, - ) - if source_revision != mapping.last_applied_source_revision + 1: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy revision has an unapplied predecessor" - ) - - association_updated = ( - "reform_label" in changed_fields and association.name != reform_label - ) - update_legacy_user_policy_state( - session, - association=association, - mapping=mapping, - reform_label=reform_label, - update_name=association_updated, - fingerprint=fingerprint, - source_revision=source_revision, - ) - return LegacyUserPolicyPersistenceResult( - association_id=association.id, - policy_id=policy_id, - association_created=False, - association_updated=association_updated, - mapping_created=False, - ) - - -def persist_legacy_user_policy_mapping( - session: Session, - *, - country_id: str, - legacy_user_policy_id: int, - reform_label: str | None, - projection: UserPolicyCreateCommand, - fingerprint: str, - source_revision: int, - changed_fields: frozenset[str], -) -> LegacyUserPolicyPersistenceResult: - """Create or advance one v1 saved-policy association mapping.""" - - existing = read_legacy_user_policy_mapping( - session, - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, - lock=True, - ) - if existing is not None: - return apply_existing_legacy_user_policy_mapping( - session, - mapping=existing, - country_id=country_id, - reform_label=reform_label, - fingerprint=fingerprint, - user_id=projection.user_id, - policy_id=projection.policy_id, - changed_fields=changed_fields, - source_revision=source_revision, - ) - - association = create_user_policy(session, projection) - mapping_id = create_legacy_user_policy_mapping( - session, - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, - user_policy_id=association.id, - source_revision=source_revision, - fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, - fingerprint=fingerprint, - ) - if mapping_id is not None: - return LegacyUserPolicyPersistenceResult( - association_id=association.id, - policy_id=projection.policy_id, - association_created=True, - association_updated=False, - mapping_created=True, - ) - - delete_user_policy(session, association) - concurrent = read_legacy_user_policy_mapping( - session, - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, - lock=False, - ) - if concurrent is None: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy mapping conflict did not resolve to a stored row" - ) - return apply_existing_legacy_user_policy_mapping( - session, - mapping=concurrent, - country_id=country_id, - reform_label=reform_label, - fingerprint=fingerprint, - user_id=projection.user_id, - policy_id=projection.policy_id, - changed_fields=changed_fields, - source_revision=source_revision, - ) - - -def persist_legacy_user_policy( - session: Session, - snapshot: LegacyUserPolicySnapshot, - reform_snapshot: LegacyPolicySnapshot, - *, - source_revision: int, - changed_fields: frozenset[str] = frozenset(), -) -> LegacyUserPolicyPersistenceResult: - """Ensure reform, association, and identity mappings in one transaction.""" - - if source_revision <= 0: - raise LegacyUserPolicyIntegrityError( - "legacy user-policy source revision must be positive" - ) - if ( - snapshot.country_id != reform_snapshot.country_id - or snapshot.reform_id != reform_snapshot.legacy_policy_id - ): - raise LegacyUserPolicyIntegrityError( - "saved policy does not reference the supplied reform snapshot" - ) - policy_result = mirror_legacy_policy_in_session(session, reform_snapshot) - user_id = resolve_legacy_user_id( - session, - legacy_user_id=snapshot.user_id, - primary_country=snapshot.country_id, - ) - projection = project_legacy_user_policy( - snapshot, - user_id=user_id, - policy_id=policy_result.policy_id, - ) - return persist_legacy_user_policy_mapping( - session, - country_id=snapshot.country_id, - legacy_user_policy_id=snapshot.legacy_user_policy_id, - reform_label=snapshot.reform_label, - projection=projection, - fingerprint=fingerprint_legacy_user_policy(snapshot), - source_revision=source_revision, - changed_fields=changed_fields, - ) diff --git a/policyengine_api/services/v2/user_policies/legacy_translation.py b/policyengine_api/services/v2/user_policies/legacy_translation.py deleted file mode 100644 index 961d26c28..000000000 --- a/policyengine_api/services/v2/user_policies/legacy_translation.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Translate committed v1 saved-policy rows into v2 association commands.""" - -from __future__ import annotations - -from hashlib import sha256 -import json -from typing import Annotated -from uuid import UUID - -from pydantic import Field - -from policyengine_api.query_parameters import CountryId, LegacyUserId -from policyengine_api.services.v2.policies.types import StrictPolicyInput -from policyengine_api.services.v2.user_policies.commands import ( - UserPolicyCreateCommand, -) - - -USER_POLICY_FINGERPRINT_VERSION = 1 - - -class LegacyUserPolicySnapshot(StrictPolicyInput): - """Detached complete committed v1 saved-policy row.""" - - country_id: CountryId - legacy_user_policy_id: Annotated[int, Field(ge=0)] - reform_id: Annotated[int, Field(ge=0)] - reform_label: Annotated[str, Field(max_length=255)] | None = None - baseline_id: Annotated[int, Field(ge=0)] - baseline_label: Annotated[str, Field(max_length=255)] | None = None - user_id: LegacyUserId - year: Annotated[str, Field(max_length=32)] - geography: Annotated[str, Field(max_length=255)] - dataset: Annotated[str, Field(max_length=255)] | None = None - number_of_provisions: Annotated[int, Field(ge=0)] - api_version: Annotated[str, Field(max_length=32)] - added_date: int - updated_date: int - budgetary_impact: Annotated[str, Field(max_length=255)] | None = None - type: Annotated[str, Field(max_length=255)] | None = None - - -def fingerprint_legacy_user_policy(snapshot: LegacyUserPolicySnapshot) -> str: - """Hash every committed source field through deterministic JSON.""" - - document = { - "fingerprint_version": USER_POLICY_FINGERPRINT_VERSION, - **snapshot.model_dump(mode="json"), - } - encoded = json.dumps( - document, - ensure_ascii=True, - allow_nan=False, - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") - return sha256(encoded).hexdigest() - - -def project_legacy_user_policy( - snapshot: LegacyUserPolicySnapshot, - *, - user_id: UUID, - policy_id: UUID, -) -> UserPolicyCreateCommand: - """Map v1 presentation data onto an association, never core policy content.""" - - return UserPolicyCreateCommand( - country_id=snapshot.country_id, - user_id=user_id, - policy_id=policy_id, - name=snapshot.reform_label, - description=None, - ) diff --git a/policyengine_api/services/v2/user_policies/service.py b/policyengine_api/services/v2/user_policies/service.py deleted file mode 100644 index 7753a6fe8..000000000 --- a/policyengine_api/services/v2/user_policies/service.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Session-owning application service for user-policy associations.""" - -from __future__ import annotations - -from uuid import UUID - -from sqlalchemy.orm import sessionmaker -from sqlmodel import Session - -from policyengine_api.data.v2.user_policies.creates import ( - create_user_policy as create_user_policy_row, -) -from policyengine_api.data.v2.user_policies.deletes import ( - delete_user_policy as delete_user_policy_row, -) -from policyengine_api.data.v2.user_policies.reads import ( - UserPolicyPage, - UserPolicyRead, - association_read, - get_user_policy_row, - list_user_policies, - read_policy_for_association, - read_user_policy, - read_user, -) -from policyengine_api.data.v2.user_policies.updates import ( - update_user_policy, -) -from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot -from policyengine_api.services.v2.user_policies.commands import ( - UserPolicyCreateCommand, - UserPolicyPatchCommand, -) -from policyengine_api.services.v2.user_policies.legacy_service import ( - LegacyUserPolicyPersistenceResult, - persist_legacy_user_policy, -) -from policyengine_api.services.v2.user_policies.legacy_translation import ( - LegacyUserPolicySnapshot, -) - - -class AssociationPolicyNotFoundError(LookupError): - """Raised when an association references an unknown policy UUID.""" - - -class AssociationUserNotFoundError(LookupError): - """Raised when an association references an unknown v2 user UUID.""" - - -class AssociationCountryConflictError(ValueError): - """Raised when an association and its referenced policy differ by country.""" - - -class V2UserPolicyService: - """Own transaction boundaries for native association operations.""" - - def __init__(self, session_factory: sessionmaker[Session]) -> None: - self._sessions = session_factory - - def create_user_policy( - self, - command: UserPolicyCreateCommand, - ) -> UserPolicyRead: - with self._sessions.begin() as session: - if read_user(session, command.user_id) is None: - raise AssociationUserNotFoundError( - f"user {command.user_id} was not found" - ) - policy = read_policy_for_association(session, command.policy_id) - if policy is None: - raise AssociationPolicyNotFoundError( - f"policy {command.policy_id} was not found" - ) - if policy.country_id != command.country_id: - raise AssociationCountryConflictError( - "Association country_id must match the referenced policy" - ) - return association_read(create_user_policy_row(session, command)) - - def get_user_policy( - self, - *, - country_id: str, - association_id: UUID, - ) -> UserPolicyRead: - with self._sessions() as session: - return read_user_policy( - session, - country_id=country_id, - association_id=association_id, - ) - - def list_user_policies( - self, - *, - country_id: str, - user_id: UUID, - policy_id: UUID | None = None, - offset: int = 0, - limit: int = 100, - ) -> UserPolicyPage: - with self._sessions() as session: - return list_user_policies( - session, - country_id=country_id, - user_id=user_id, - policy_id=policy_id, - offset=offset, - limit=limit, - ) - - def patch_user_policy( - self, - *, - country_id: str, - association_id: UUID, - command: UserPolicyPatchCommand, - ) -> UserPolicyRead: - with self._sessions.begin() as session: - association = get_user_policy_row( - session, - country_id=country_id, - association_id=association_id, - ) - return association_read(update_user_policy(session, association, command)) - - def delete_user_policy( - self, - *, - country_id: str, - association_id: UUID, - ) -> None: - with self._sessions.begin() as session: - association = get_user_policy_row( - session, - country_id=country_id, - association_id=association_id, - ) - delete_user_policy_row(session, association) - - def mirror_legacy_user_policy( - self, - snapshot: LegacyUserPolicySnapshot, - reform_snapshot: LegacyPolicySnapshot, - *, - source_revision: int, - changed_fields: frozenset[str], - ) -> LegacyUserPolicyPersistenceResult: - """Mirror one committed v1 saved policy in one Supabase transaction.""" - - with self._sessions.begin() as session: - return persist_legacy_user_policy( - session, - snapshot, - reform_snapshot, - source_revision=source_revision, - changed_fields=changed_fields, - ) diff --git a/policyengine_api/services/v2/user_policies/services.py b/policyengine_api/services/v2/user_policies/services.py new file mode 100644 index 000000000..ac71912d4 --- /dev/null +++ b/policyengine_api/services/v2/user_policies/services.py @@ -0,0 +1,394 @@ +"""Router-facing and legacy-mirroring services for v2 user-policy resources.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session + +from policyengine_api.data.v2.models import LegacyUserPolicyMapping +from policyengine_api.services.v2.policies.services import ( + mirror_legacy_policy_in_session, +) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.user_policies.database_connectors.creates import ( + create_legacy_user_mapping, + create_legacy_user_policy_mapping, + create_transition_user, + create_user_policy, +) +from policyengine_api.services.v2.user_policies.database_connectors.deletes import ( + delete_transition_user, + delete_user_policy, +) +from policyengine_api.services.v2.user_policies.database_connectors.reads import ( + read_legacy_user_mapping, + read_legacy_user_policy_mapping, + read_mapped_user_policy, + read_policy_for_association, + read_user, + read_user_policy_row, + read_user_policy_rows, +) +from policyengine_api.services.v2.user_policies.database_connectors.updates import ( + update_legacy_user_policy_state, + update_user_policy, +) +from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, +) +from policyengine_api.services.v2.user_policies.transformations import ( + USER_POLICY_FINGERPRINT_VERSION, + fingerprint_legacy_user_policy, + legacy_name_requires_update, + project_legacy_user_policy, + user_policy_page, + user_policy_read, +) +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyMappingAction, + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, + UserPolicyCreationInput, + UserPolicyPage, + UserPolicyRead, + UserPolicyUpdateInput, +) +from policyengine_api.services.v2.user_policies.validators import ( + LegacyUserPolicyIntegrityError, + require_user_policy, + validate_association_creation, + validate_existing_legacy_user_mapping, + validate_existing_legacy_user_policy_mapping, + validate_legacy_user_mapping_conflict, + validate_legacy_user_policy_input, +) + + +def read_complete_user_policy( + session: Session, *, country_id: str, association_id: UUID +) -> UserPolicyRead: + association = require_user_policy( + read_user_policy_row( + session, + country_id=country_id, + association_id=association_id, + ), + association_id=association_id, + ) + return user_policy_read(association) + + +def read_user_policy_page( + session: Session, + *, + country_id: str, + user_id: UUID, + policy_id: UUID | None, + offset: int, + limit: int, +) -> UserPolicyPage: + rows = read_user_policy_rows( + session, + country_id=country_id, + user_id=user_id, + policy_id=policy_id, + offset=offset, + limit=limit, + ) + return user_policy_page(rows, offset=offset, limit=limit) + + +def resolve_legacy_user_id( + session: Session, + *, + legacy_user_id: str, + primary_country: str, +) -> UUID: + existing = read_legacy_user_mapping(session, legacy_user_id, lock=True) + if existing is not None: + return validate_existing_legacy_user_mapping( + existing, + user=read_user(session, existing.user_id), + ) + + user = create_transition_user(session, primary_country=primary_country) + created_user_id = create_legacy_user_mapping( + session, + legacy_user_id=legacy_user_id, + user_id=user.id, + ) + if created_user_id is not None: + return created_user_id + + delete_transition_user(session, user) + concurrent = read_legacy_user_mapping(session, legacy_user_id, lock=False) + return validate_legacy_user_mapping_conflict( + concurrent, + user=( + read_user(session, concurrent.user_id) if concurrent is not None else None + ), + ) + + +def apply_existing_legacy_user_policy_mapping( + session: Session, + *, + mapping: LegacyUserPolicyMapping, + country_id: str, + reform_label: str | None, + fingerprint: str, + user_id: UUID, + policy_id: UUID, + changed_fields: frozenset[str], + source_revision: int, +) -> LegacyUserPolicyPersistenceResult: + action, association = validate_existing_legacy_user_policy_mapping( + mapping, + association=read_mapped_user_policy(session, mapping), + country_id=country_id, + fingerprint=fingerprint, + user_id=user_id, + policy_id=policy_id, + source_revision=source_revision, + fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, + ) + if action in { + LegacyUserPolicyMappingAction.STALE, + LegacyUserPolicyMappingAction.REPLAY, + }: + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=False, + mapping_created=False, + ) + + association_updated = legacy_name_requires_update( + association, + reform_label=reform_label, + changed_fields=changed_fields, + ) + update_legacy_user_policy_state( + session, + association=association, + mapping=mapping, + reform_label=reform_label, + update_name=association_updated, + fingerprint=fingerprint, + source_revision=source_revision, + ) + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=association_updated, + mapping_created=False, + ) + + +def persist_legacy_user_policy_mapping( + session: Session, + *, + country_id: str, + legacy_user_policy_id: int, + reform_label: str | None, + projection: UserPolicyCreationInput, + fingerprint: str, + source_revision: int, + changed_fields: frozenset[str], +) -> LegacyUserPolicyPersistenceResult: + existing = read_legacy_user_policy_mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + lock=True, + ) + if existing is not None: + return apply_existing_legacy_user_policy_mapping( + session, + mapping=existing, + country_id=country_id, + reform_label=reform_label, + fingerprint=fingerprint, + user_id=projection.user_id, + policy_id=projection.policy_id, + changed_fields=changed_fields, + source_revision=source_revision, + ) + + association = create_user_policy(session, projection) + mapping_id = create_legacy_user_policy_mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + user_policy_id=association.id, + source_revision=source_revision, + fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, + fingerprint=fingerprint, + ) + if mapping_id is not None: + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=projection.policy_id, + association_created=True, + association_updated=False, + mapping_created=True, + ) + + delete_user_policy(session, association) + concurrent = read_legacy_user_policy_mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + lock=False, + ) + if concurrent is None: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping conflict did not resolve to a stored row" + ) + return apply_existing_legacy_user_policy_mapping( + session, + mapping=concurrent, + country_id=country_id, + reform_label=reform_label, + fingerprint=fingerprint, + user_id=projection.user_id, + policy_id=projection.policy_id, + changed_fields=changed_fields, + source_revision=source_revision, + ) + + +def mirror_legacy_user_policy_in_session( + session: Session, + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str] = frozenset(), +) -> LegacyUserPolicyPersistenceResult: + validate_legacy_user_policy_input( + snapshot, + reform_snapshot, + source_revision=source_revision, + ) + policy_result = mirror_legacy_policy_in_session(session, reform_snapshot) + user_id = resolve_legacy_user_id( + session, + legacy_user_id=snapshot.user_id, + primary_country=snapshot.country_id, + ) + projection = project_legacy_user_policy( + snapshot, + user_id=user_id, + policy_id=policy_result.policy_id, + ) + return persist_legacy_user_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + reform_label=snapshot.reform_label, + projection=projection, + fingerprint=fingerprint_legacy_user_policy(snapshot), + source_revision=source_revision, + changed_fields=changed_fields, + ) + + +class V2UserPolicyService: + """Sequence association operations through explicit database sessions.""" + + def __init__(self, database_session: UserPolicyDatabaseSession) -> None: + self._database_session = database_session + + def create_user_policy( + self, association_input: UserPolicyCreationInput + ) -> UserPolicyRead: + with self._database_session.transaction() as session: + policy = read_policy_for_association(session, association_input.policy_id) + validate_association_creation( + association_input, + user=read_user(session, association_input.user_id), + policy=policy, + ) + return user_policy_read(create_user_policy(session, association_input)) + + def get_user_policy( + self, *, country_id: str, association_id: UUID + ) -> UserPolicyRead: + with self._database_session.read() as session: + return read_complete_user_policy( + session, + country_id=country_id, + association_id=association_id, + ) + + def list_user_policies( + self, + *, + country_id: str, + user_id: UUID, + policy_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> UserPolicyPage: + with self._database_session.read() as session: + return read_user_policy_page( + session, + country_id=country_id, + user_id=user_id, + policy_id=policy_id, + offset=offset, + limit=limit, + ) + + def patch_user_policy( + self, + *, + country_id: str, + association_id: UUID, + association_input: UserPolicyUpdateInput, + ) -> UserPolicyRead: + with self._database_session.transaction() as session: + association = require_user_policy( + read_user_policy_row( + session, + country_id=country_id, + association_id=association_id, + ), + association_id=association_id, + ) + return user_policy_read( + update_user_policy(session, association, association_input) + ) + + def delete_user_policy(self, *, country_id: str, association_id: UUID) -> None: + with self._database_session.transaction() as session: + association = require_user_policy( + read_user_policy_row( + session, + country_id=country_id, + association_id=association_id, + ), + association_id=association_id, + ) + delete_user_policy(session, association) + + def mirror_legacy_user_policy( + self, + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str], + ) -> LegacyUserPolicyPersistenceResult: + with self._database_session.transaction() as session: + return mirror_legacy_user_policy_in_session( + session, + snapshot, + reform_snapshot, + source_revision=source_revision, + changed_fields=changed_fields, + ) diff --git a/policyengine_api/services/v2/user_policies/transformations.py b/policyengine_api/services/v2/user_policies/transformations.py new file mode 100644 index 000000000..615059f07 --- /dev/null +++ b/policyengine_api/services/v2/user_policies/transformations.py @@ -0,0 +1,85 @@ +"""Pure representation transformations for v2 user-policy operations.""" + +from __future__ import annotations + +from hashlib import sha256 +import json +from uuid import UUID + +from policyengine_api.data.v2.models import UserPolicy +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicySnapshot, + UserPolicyCreationInput, + UserPolicyPage, + UserPolicyRead, +) + + +USER_POLICY_FINGERPRINT_VERSION = 1 + + +def user_policy_read(association: UserPolicy) -> UserPolicyRead: + return UserPolicyRead( + id=association.id, + country_id=association.country_id, + user_id=association.user_id, + policy_id=association.policy_id, + name=association.name, + description=association.description, + created_at=association.created_at, + updated_at=association.updated_at, + ) + + +def user_policy_page( + rows: list[UserPolicy], *, offset: int, limit: int +) -> UserPolicyPage: + return UserPolicyPage( + items=tuple(user_policy_read(row) for row in rows[:limit]), + offset=offset, + limit=limit, + has_more=len(rows) > limit, + ) + + +def fingerprint_legacy_user_policy(snapshot: LegacyUserPolicySnapshot) -> str: + """Hash every committed source field through deterministic JSON.""" + + document = { + "fingerprint_version": USER_POLICY_FINGERPRINT_VERSION, + **snapshot.model_dump(mode="json"), + } + encoded = json.dumps( + document, + ensure_ascii=True, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def project_legacy_user_policy( + snapshot: LegacyUserPolicySnapshot, + *, + user_id: UUID, + policy_id: UUID, +) -> UserPolicyCreationInput: + """Map v1 presentation data onto an association, never core policy content.""" + + return UserPolicyCreationInput( + country_id=snapshot.country_id, + user_id=user_id, + policy_id=policy_id, + name=snapshot.reform_label, + description=None, + ) + + +def legacy_name_requires_update( + association: UserPolicy, + *, + reform_label: str | None, + changed_fields: frozenset[str], +) -> bool: + return "reform_label" in changed_fields and association.name != reform_label diff --git a/policyengine_api/services/v2/user_policies/types.py b/policyengine_api/services/v2/user_policies/types.py new file mode 100644 index 000000000..67f70cc6a --- /dev/null +++ b/policyengine_api/services/v2/user_policies/types.py @@ -0,0 +1,99 @@ +"""Framework-independent data exchanged by v2 user-policy layers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from enum import StrEnum +from typing import Annotated +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator + +from policyengine_api.query_parameters import ( + CountryId, + LegacyUserId, + ResourceId, + UserId, +) + + +class StrictAssociationInput(BaseModel): + """Reject fields outside the reviewed association contract.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class UserPolicyCreationInput(StrictAssociationInput): + country_id: CountryId + user_id: UserId + policy_id: ResourceId + name: Annotated[str, StringConstraints(max_length=255)] | None = None + description: str | None = None + + +class UserPolicyUpdateInput(StrictAssociationInput): + name: Annotated[str, StringConstraints(max_length=255)] | None = None + description: str | None = None + + @model_validator(mode="after") + def require_supplied_field(self) -> "UserPolicyUpdateInput": + if not self.model_fields_set.intersection({"name", "description"}): + raise ValueError("At least one of name or description must be supplied") + return self + + +class LegacyUserPolicySnapshot(StrictAssociationInput): + """Detached complete committed v1 saved-policy row.""" + + country_id: CountryId + legacy_user_policy_id: Annotated[int, Field(ge=0)] + reform_id: Annotated[int, Field(ge=0)] + reform_label: Annotated[str, Field(max_length=255)] | None = None + baseline_id: Annotated[int, Field(ge=0)] + baseline_label: Annotated[str, Field(max_length=255)] | None = None + user_id: LegacyUserId + year: Annotated[str, Field(max_length=32)] + geography: Annotated[str, Field(max_length=255)] + dataset: Annotated[str, Field(max_length=255)] | None = None + number_of_provisions: Annotated[int, Field(ge=0)] + api_version: Annotated[str, Field(max_length=32)] + added_date: int + updated_date: int + budgetary_impact: Annotated[str, Field(max_length=255)] | None = None + type: Annotated[str, Field(max_length=255)] | None = None + + +class LegacyUserPolicyMappingAction(StrEnum): + STALE = "stale" + REPLAY = "replay" + UPDATE = "update" + + +@dataclass(frozen=True) +class UserPolicyRead: + id: UUID + country_id: str + user_id: UUID + policy_id: UUID + name: str | None + description: str | None + created_at: datetime + updated_at: datetime + + +@dataclass(frozen=True) +class UserPolicyPage: + items: tuple[UserPolicyRead, ...] + offset: int + limit: int + has_more: bool + + +@dataclass(frozen=True) +class LegacyUserPolicyPersistenceResult: + association_id: UUID + policy_id: UUID + association_created: bool + association_updated: bool + mapping_created: bool diff --git a/policyengine_api/services/v2/user_policies/validators.py b/policyengine_api/services/v2/user_policies/validators.py new file mode 100644 index 000000000..1d895ecba --- /dev/null +++ b/policyengine_api/services/v2/user_policies/validators.py @@ -0,0 +1,155 @@ +"""Database-independent validation for v2 user-policy operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from uuid import UUID + +from policyengine_api.data.v2.models import ( + LegacyUserMapping, + LegacyUserPolicyMapping, + Policy, + User, + UserPolicy, +) +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyMappingAction, +) + +if TYPE_CHECKING: + from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot + from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicySnapshot, + UserPolicyCreationInput, + ) + + +class UserPolicyNotFoundError(LookupError): + """Raised when an association is absent from the selected country.""" + + +class AssociationPolicyNotFoundError(LookupError): + """Raised when an association references an unknown policy UUID.""" + + +class AssociationUserNotFoundError(LookupError): + """Raised when an association references an unknown v2 user UUID.""" + + +class AssociationCountryConflictError(ValueError): + """Raised when an association and its referenced policy differ by country.""" + + +class LegacyUserPolicyIntegrityError(RuntimeError): + """Raised when source, policy, association, or mapping identity conflicts.""" + + +def require_user_policy( + association: UserPolicy | None, *, association_id: UUID +) -> UserPolicy: + if association is None: + raise UserPolicyNotFoundError( + f"user-policy association {association_id} was not found" + ) + return association + + +def validate_association_creation( + association_input: "UserPolicyCreationInput", + *, + user: User | None, + policy: Policy | None, +) -> None: + if user is None: + raise AssociationUserNotFoundError( + f"user {association_input.user_id} was not found" + ) + if policy is None: + raise AssociationPolicyNotFoundError( + f"policy {association_input.policy_id} was not found" + ) + if policy.country_id != association_input.country_id: + raise AssociationCountryConflictError( + "Association country_id must match the referenced policy" + ) + + +def validate_existing_legacy_user_mapping( + mapping: LegacyUserMapping, *, user: User | None +) -> UUID: + if user is None: + raise LegacyUserPolicyIntegrityError( + "legacy user mapping has no referenced v2 user" + ) + return mapping.user_id + + +def validate_legacy_user_mapping_conflict( + mapping: LegacyUserMapping | None, *, user: User | None +) -> UUID: + if mapping is None or user is None: + raise LegacyUserPolicyIntegrityError( + "legacy user mapping conflict did not resolve to a v2 user" + ) + return mapping.user_id + + +def validate_legacy_user_policy_input( + snapshot: "LegacyUserPolicySnapshot", + reform_snapshot: "LegacyPolicySnapshot", + *, + source_revision: int, +) -> None: + if source_revision <= 0: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy source revision must be positive" + ) + if ( + snapshot.country_id != reform_snapshot.country_id + or snapshot.reform_id != reform_snapshot.legacy_policy_id + ): + raise LegacyUserPolicyIntegrityError( + "saved policy does not reference the supplied reform snapshot" + ) + + +def validate_existing_legacy_user_policy_mapping( + mapping: LegacyUserPolicyMapping, + *, + association: UserPolicy | None, + country_id: str, + fingerprint: str, + user_id: UUID, + policy_id: UUID, + source_revision: int, + fingerprint_version: int, +) -> tuple[LegacyUserPolicyMappingAction, UserPolicy]: + if association is None: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping has no association" + ) + if ( + association.policy_id != policy_id + or association.country_id != country_id + or association.user_id != user_id + ): + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping conflicts with immutable association fields" + ) + if mapping.fingerprint_version != fingerprint_version: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy fingerprint version is unsupported" + ) + if source_revision < mapping.last_applied_source_revision: + return LegacyUserPolicyMappingAction.STALE, association + if source_revision == mapping.last_applied_source_revision: + if mapping.fingerprint_sha256 != fingerprint: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy revision conflicts with its stored fingerprint" + ) + return LegacyUserPolicyMappingAction.REPLAY, association + if source_revision != mapping.last_applied_source_revision + 1: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy revision has an unapplied predecessor" + ) + return LegacyUserPolicyMappingAction.UPDATE, association diff --git a/pyproject.toml b/pyproject.toml index c79dc0b1a..f5e873fbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,6 @@ files = [ "policyengine_api/fastapi_routes/query_parameters.py", "policyengine_api/fastapi_routes/dependencies.py", "policyengine_api/fastapi_routes/v2", - "policyengine_api/data/v2/user_policies", "policyengine_api/services/v2", ] explicit_package_bases = true diff --git a/tests/contract/test_policy_v2_compatibility.py b/tests/contract/test_policy_v2_compatibility.py index 3a2bd7f39..b8bfaec68 100644 --- a/tests/contract/test_policy_v2_compatibility.py +++ b/tests/contract/test_policy_v2_compatibility.py @@ -10,10 +10,10 @@ from policyengine_api.fastapi_routes.v2.policies.request_models import ( PolicyCreateRequest, ) -from policyengine_api.services.v2.user_policies.legacy_translation import ( - LegacyUserPolicySnapshot, +from policyengine_api.services.v2.user_policies.transformations import ( project_legacy_user_policy, ) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") diff --git a/tests/contract/test_v1_route_contracts.py b/tests/contract/test_v1_route_contracts.py index 2a400ad1b..2296369a4 100644 --- a/tests/contract/test_v1_route_contracts.py +++ b/tests/contract/test_v1_route_contracts.py @@ -16,9 +16,7 @@ Simulation, UserPolicy, ) -from policyengine_api.services.v2.user_policies.legacy_translation import ( - LegacyUserPolicySnapshot, -) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot from policyengine_api.extensions import cache from policyengine_api.routes.household_routes import household_bp from policyengine_api.routes.policy_routes import policy_bp diff --git a/tests/integration/test_v1_user_policy_dual_write.py b/tests/integration/test_v1_user_policy_dual_write.py index 4fcd6f5a2..a57819f52 100644 --- a/tests/integration/test_v1_user_policy_dual_write.py +++ b/tests/integration/test_v1_user_policy_dual_write.py @@ -28,7 +28,10 @@ UserPolicy, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL -from policyengine_api.services.v2.user_policies.service import V2UserPolicyService +from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, +) +from policyengine_api.services.v2.user_policies.services import V2UserPolicyService from policyengine_api.services.policy_service import PolicyService from policyengine_api.services.user_policy_mirroring import ( UserPolicyMirrorUnavailableError, @@ -216,7 +219,7 @@ def test_create_update_and_v1_only_change_mirror_one_association() -> None: _saved_values(reform.policy_id), record_mirror_event=True, ) - mirror_service = V2UserPolicyService(v2_sessions) + mirror_service = V2UserPolicyService(UserPolicyDatabaseSession(v2_sessions)) first = mirror_pending_user_policy_events_after_commit( "us", created.user_policy.id, @@ -296,7 +299,7 @@ def test_failure_after_cloud_commit_and_identical_create_retry_are_idempotent() _saved_values(reform.policy_id), record_mirror_event=True, ) - mirror_service = V2UserPolicyService(v2_sessions) + mirror_service = V2UserPolicyService(UserPolicyDatabaseSession(v2_sessions)) with pytest.raises(UserPolicyMirrorUnavailableError): mirror_pending_user_policy_events_after_commit( @@ -379,7 +382,7 @@ def test_destination_commit_replays_when_source_processing_marker_is_missing() - _saved_values(reform.policy_id), record_mirror_event=True, ) - mirror_service = V2UserPolicyService(v2_sessions) + mirror_service = V2UserPolicyService(UserPolicyDatabaseSession(v2_sessions)) committed = mirror_user_policy_after_commit( created.snapshot, diff --git a/tests/integration/test_v2_user_policy_mirroring.py b/tests/integration/test_v2_user_policy_mirroring.py index 6610946f5..1b691451f 100644 --- a/tests/integration/test_v2_user_policy_mirroring.py +++ b/tests/integration/test_v2_user_policy_mirroring.py @@ -29,18 +29,16 @@ User, UserPolicy, ) -from policyengine_api.services.v2.user_policies.legacy_service import ( - resolve_legacy_user_id, -) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot -from policyengine_api.services.v2.user_policies.legacy_service import ( - persist_legacy_user_policy, +from policyengine_api.services.v2.user_policies.services import ( + mirror_legacy_user_policy_in_session, + resolve_legacy_user_id, ) -from policyengine_api.services.v2.user_policies.legacy_translation import ( - LegacyUserPolicySnapshot, +from policyengine_api.services.v2.user_policies.transformations import ( fingerprint_legacy_user_policy, ) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot def _disposable_url() -> str: @@ -197,26 +195,26 @@ def test_saved_rows_share_policy_and_reuse_only_the_same_mapped_user() -> None: ) with sessions.begin() as session: - first = persist_legacy_user_policy( + first = mirror_legacy_user_policy_in_session( session, first_saved, first_reform, source_revision=1, ) - second = persist_legacy_user_policy( + second = mirror_legacy_user_policy_in_session( session, second_saved, second_reform, source_revision=1, ) - third = persist_legacy_user_policy( + third = mirror_legacy_user_policy_in_session( session, third_saved, first_reform, source_revision=1, ) with sessions.begin() as session: - retry = persist_legacy_user_policy( + retry = mirror_legacy_user_policy_in_session( session, first_saved, first_reform, @@ -317,7 +315,7 @@ def test_label_and_v1_only_updates_advance_the_complete_row_fingerprint() -> Non reform = _reform(parameter_name, legacy_id=301, source_hash="reform") original = _saved(legacy_id=401, reform_id=301) with sessions.begin() as session: - created = persist_legacy_user_policy( + created = mirror_legacy_user_policy_in_session( session, original, reform, @@ -331,7 +329,7 @@ def test_label_and_v1_only_updates_advance_the_complete_row_fingerprint() -> Non association = session.get(UserPolicy, created.association_id) association.description = "Native description" with sessions.begin() as session: - rename_result = persist_legacy_user_policy( + rename_result = mirror_legacy_user_policy_in_session( session, renamed, reform, @@ -359,7 +357,7 @@ def test_label_and_v1_only_updates_advance_the_complete_row_fingerprint() -> Non created.association_id, ).updated_at with sessions.begin() as session: - v1_only_result = persist_legacy_user_policy( + v1_only_result = mirror_legacy_user_policy_in_session( session, v1_only, reform, @@ -395,7 +393,7 @@ def test_complete_transaction_rolls_back_and_native_delete_is_isolated() -> None saved = _saved(legacy_id=601, reform_id=501) with pytest.raises(RuntimeError, match="forced rollback"): with sessions.begin() as session: - persist_legacy_user_policy( + mirror_legacy_user_policy_in_session( session, saved, reform, @@ -422,7 +420,7 @@ def test_complete_transaction_rolls_back_and_native_delete_is_isolated() -> None ) with sessions.begin() as session: - created = persist_legacy_user_policy( + created = mirror_legacy_user_policy_in_session( session, saved, reform, diff --git a/tests/unit/routes/test_user_policy_dual_write_routes.py b/tests/unit/routes/test_user_policy_dual_write_routes.py index 420f867a4..7f003a8b5 100644 --- a/tests/unit/routes/test_user_policy_dual_write_routes.py +++ b/tests/unit/routes/test_user_policy_dual_write_routes.py @@ -16,9 +16,7 @@ from policyengine_api.data.v1_models import UserPolicy from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot -from policyengine_api.services.v2.user_policies.legacy_translation import ( - LegacyUserPolicySnapshot, -) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot from policyengine_api.routes.policy_routes import policy_bp from policyengine_api.services.user_policy_mirroring import ( UserPolicyMirrorUnavailableError, diff --git a/tests/unit/services/test_user_policy_mirroring.py b/tests/unit/services/test_user_policy_mirroring.py index 2b59f8bc0..dc60d6fcf 100644 --- a/tests/unit/services/test_user_policy_mirroring.py +++ b/tests/unit/services/test_user_policy_mirroring.py @@ -10,14 +10,14 @@ from sqlalchemy.exc import OperationalError, TimeoutError from policyengine_api.data.v1_models import UserPolicyMirrorEvent -from policyengine_api.services.v2.user_policies.legacy_service import ( - LegacyUserPolicyIntegrityError, - LegacyUserPolicyPersistenceResult, -) from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot -from policyengine_api.services.v2.user_policies.legacy_translation import ( +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyPersistenceResult, LegacyUserPolicySnapshot, ) +from policyengine_api.services.v2.user_policies.validators import ( + LegacyUserPolicyIntegrityError, +) from policyengine_api.services.user_policy_mirroring import ( UserPolicyMirrorUnavailableError, mirror_pending_user_policy_events_after_commit, diff --git a/tests/unit/services/test_user_policy_service.py b/tests/unit/services/test_user_policy_service.py index e6aaf9f0f..51d361138 100644 --- a/tests/unit/services/test_user_policy_service.py +++ b/tests/unit/services/test_user_policy_service.py @@ -14,7 +14,7 @@ ) from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent -from policyengine_api.services.v2.user_policies.legacy_service import ( +from policyengine_api.services.v2.user_policies.types import ( LegacyUserPolicyPersistenceResult, ) from policyengine_api.services.user_policy_service import ( diff --git a/tests/unit/v2/test_data_crud_boundaries.py b/tests/unit/v2/test_data_crud_boundaries.py index 581c2f881..fcef5b1b1 100644 --- a/tests/unit/v2/test_data_crud_boundaries.py +++ b/tests/unit/v2/test_data_crud_boundaries.py @@ -9,7 +9,6 @@ PROJECT_ROOT = Path(__file__).parents[3] -DATA_ROOT = PROJECT_ROOT / "policyengine_api" / "data" / "v2" SERVICES_ROOT = PROJECT_ROOT / "policyengine_api" / "services" / "v2" @@ -42,26 +41,14 @@ def _imported_modules(path: Path) -> set[str]: @pytest.mark.parametrize( "relative_path", ( - "user_policies/creates.py", - "user_policies/updates.py", - "user_policies/deletes.py", + "policies/database_connectors/creates.py", + "user_policies/database_connectors/creates.py", + "user_policies/database_connectors/updates.py", + "user_policies/database_connectors/deletes.py", ), ) -def test_existing_mutation_modules_contain_no_database_reads( - relative_path: str, -) -> None: - calls = _called_names(DATA_ROOT / relative_path) - assert "select" not in calls - assert "get" not in calls - - -def test_existing_read_module_contains_no_database_mutations() -> None: - calls = _called_names(DATA_ROOT / "user_policies/reads.py") - assert calls.isdisjoint({"insert", "add", "add_all", "delete"}) - - -def test_policy_create_connectors_contain_no_database_reads() -> None: - calls = _called_names(SERVICES_ROOT / "policies/database_connectors/creates.py") +def test_mutation_connectors_contain_no_database_reads(relative_path: str) -> None: + calls = _called_names(SERVICES_ROOT / relative_path) assert "select" not in calls assert "get" not in calls @@ -70,6 +57,7 @@ def test_policy_create_connectors_contain_no_database_reads() -> None: "relative_path", ( "policies/database_connectors/reads.py", + "user_policies/database_connectors/reads.py", "metadata/database_connectors/reads.py", "metadata/database_connectors/reads_datasets.py", "metadata/database_connectors/reads_parameter_tree.py", @@ -90,6 +78,8 @@ def test_read_connectors_contain_no_database_mutations(relative_path: str) -> No "policies/transformations.py", "metadata/validators.py", "metadata/transformations.py", + "user_policies/validators.py", + "user_policies/transformations.py", ), ) def test_validation_and_transformation_modules_have_no_database_query_dependency( @@ -107,7 +97,11 @@ def test_validation_and_transformation_modules_have_no_database_query_dependency @pytest.mark.parametrize( "relative_path", - ("policies/database_session.py", "metadata/database_session.py"), + ( + "policies/database_session.py", + "metadata/database_session.py", + "user_policies/database_session.py", + ), ) def test_database_session_modules_do_not_construct_queries( relative_path: str, diff --git a/tests/unit/v2/test_user_policy_legacy.py b/tests/unit/v2/test_user_policy_legacy.py index 95e606432..621ff217e 100644 --- a/tests/unit/v2/test_user_policy_legacy.py +++ b/tests/unit/v2/test_user_policy_legacy.py @@ -8,15 +8,17 @@ import pytest from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy -from policyengine_api.services.v2.user_policies.legacy_service import ( - LegacyUserPolicyIntegrityError, +from policyengine_api.services.v2.user_policies.services import ( apply_existing_legacy_user_policy_mapping, ) -from policyengine_api.services.v2.user_policies.legacy_translation import ( +from policyengine_api.services.v2.user_policies.transformations import ( USER_POLICY_FINGERPRINT_VERSION, - LegacyUserPolicySnapshot, fingerprint_legacy_user_policy, ) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot +from policyengine_api.services.v2.user_policies.validators import ( + LegacyUserPolicyIntegrityError, +) POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py index f9eea38d5..8d12d6ca9 100644 --- a/tests/unit/v2/test_user_policy_routes.py +++ b/tests/unit/v2/test_user_policy_routes.py @@ -12,15 +12,15 @@ from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.data.v2.settings import V2ConfigurationError -from policyengine_api.data.v2.user_policies.reads import ( - UserPolicyNotFoundError, +from policyengine_api.services.v2.user_policies.types import ( UserPolicyPage, UserPolicyRead, ) -from policyengine_api.services.v2.user_policies.service import ( +from policyengine_api.services.v2.user_policies.validators import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, + UserPolicyNotFoundError, ) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies from policyengine_api.migration_flags import ( @@ -66,8 +66,8 @@ def _raise(self) -> None: if self.error is not None: raise self.error - def create_user_policy(self, command) -> UserPolicyRead: - self.calls.append(("create_user_policy", command)) + def create_user_policy(self, association_input) -> UserPolicyRead: + self.calls.append(("create_user_policy", association_input)) self._raise() return self.next_item @@ -89,11 +89,15 @@ def list_user_policies(self, **filters) -> UserPolicyPage: def patch_user_policy(self, **changes) -> UserPolicyRead: self.calls.append(("patch_user_policy", changes)) self._raise() - command = changes["command"] - name = command.name if "name" in command.model_fields_set else "Saved reform" + association_input = changes["association_input"] + name = ( + association_input.name + if "name" in association_input.model_fields_set + else "Saved reform" + ) description = ( - command.description - if "description" in command.model_fields_set + association_input.description + if "description" in association_input.model_fields_set else "Personal note" ) return _association_read(name=name, description=description) diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py index 7653a110a..3444bf9ae 100644 --- a/tests/unit/v2/test_user_policy_service.py +++ b/tests/unit/v2/test_user_policy_service.py @@ -21,19 +21,20 @@ UserPolicy, V2_METADATA, ) -from policyengine_api.data.v2.user_policies.reads import ( - UserPolicyNotFoundError, +from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, +) +from policyengine_api.services.v2.user_policies.services import V2UserPolicyService +from policyengine_api.services.v2.user_policies.types import ( + UserPolicyCreationInput, + UserPolicyUpdateInput, ) -from policyengine_api.services.v2.user_policies.service import ( +from policyengine_api.services.v2.user_policies.validators import ( AssociationCountryConflictError, AssociationPolicyNotFoundError, AssociationUserNotFoundError, + UserPolicyNotFoundError, ) -from policyengine_api.services.v2.user_policies.commands import ( - UserPolicyCreateCommand, - UserPolicyPatchCommand, -) -from policyengine_api.services.v2.user_policies.service import V2UserPolicyService USER_ID = UUID("00000000-0000-0000-0000-000000000070") @@ -87,11 +88,11 @@ def enable_foreign_keys(dbapi_connection, _connection_record) -> None: session.flush() identity = (policy.id, value.id) - yield V2UserPolicyService(sessions), sessions, identity + yield V2UserPolicyService(UserPolicyDatabaseSession(sessions)), sessions, identity engine.dispose() -def _command(policy_id, **changes) -> UserPolicyCreateCommand: +def _command(policy_id, **changes) -> UserPolicyCreationInput: values = { "country_id": "us", "user_id": USER_ID, @@ -100,7 +101,7 @@ def _command(policy_id, **changes) -> UserPolicyCreateCommand: "description": "Personal note", } values.update(changes) - return UserPolicyCreateCommand.model_validate(values) + return UserPolicyCreationInput.model_validate(values) def test_create_allows_distinct_duplicate_links_for_an_existing_user( @@ -178,12 +179,12 @@ def test_patch_changes_only_supplied_fields_and_supports_null_clearing( renamed = service.patch_user_policy( country_id="us", association_id=created.id, - command=UserPolicyPatchCommand(name="Renamed"), + association_input=UserPolicyUpdateInput(name="Renamed"), ) cleared = service.patch_user_policy( country_id="us", association_id=created.id, - command=UserPolicyPatchCommand(description=None), + association_input=UserPolicyUpdateInput(description=None), ) assert renamed.name == "Renamed" @@ -226,8 +227,8 @@ def test_delete_removes_mapping_but_preserves_policy_and_parameter_value( def test_patch_command_rejects_empty_and_immutable_fields() -> None: with pytest.raises(ValueError): - UserPolicyPatchCommand() + UserPolicyUpdateInput() with pytest.raises(ValueError): - UserPolicyPatchCommand.model_validate( + UserPolicyUpdateInput.model_validate( {"policy_id": "00000000-0000-0000-0000-000000000001"} ) From 289031d77561dad0743c01093ad4f823c3601aa1 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:01:42 +0400 Subject: [PATCH 13/18] Unify API v2 error responses --- .../skills/v2-code-organization.md | 6 +++ policyengine_api/asgi_factory.py | 30 ++++----------- policyengine_api/fastapi_routes/v2/errors.py | 29 +++++++++++++++ .../fastapi_routes/v2/metadata/common.py | 37 ++++++++----------- .../v2/metadata/response_models.py | 9 +---- .../v2/policies/response_models.py | 24 +++++------- .../fastapi_routes/v2/policies/routes.py | 37 +++++++------------ policyengine_api/fastapi_routes/v2/routes.py | 32 ++++++++-------- .../v2/user_policies/response_models.py | 22 +++++------ .../fastapi_routes/v2/user_policies/routes.py | 20 +++------- tests/integration/test_live_v2_metadata.py | 2 +- tests/unit/v2/test_metadata_routes.py | 10 ++++- tests/unit/v2/test_policy_routes.py | 8 ++++ tests/unit/v2/test_user_policy_routes.py | 4 ++ 14 files changed, 134 insertions(+), 136 deletions(-) create mode 100644 policyengine_api/fastapi_routes/v2/errors.py diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index dd29a98ef..569e1cc41 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -9,6 +9,7 @@ Service modules must sequence work but must not construct or execute SQL. Resource-specific FastAPI code lives under `policyengine_api/fastapi_routes/v2/`: ```text +errors.py policies/ request_models.py response_models.py @@ -23,6 +24,11 @@ metadata/ *_routes.py ``` +`errors.py` defines the strict error envelope and serialization function shared +by every API v2 resource. Application-wide exception handling distinguishes +API v2 from non-v2 requests but must not select error behavior by matching +individual resource paths. + Request models describe HTTP bodies. Response models describe public response envelopes and OpenAPI output. Route functions handle HTTP-only conditions, invoke one service method, and convert typed failures to HTTP responses. diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index a7700246c..c11eb1a78 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -19,14 +19,11 @@ from policyengine_api.fastapi_routes.specification import ( build_specification_router, ) -from policyengine_api.fastapi_routes.v2.policies.routes import ( - PolicyRequestTooLargeError, - policy_error_response, +from policyengine_api.fastapi_routes.v2.errors import ( + V2RequestTooLargeError, + v2_error_response, ) from policyengine_api.fastapi_routes.v2.routes import build_v2_router -from policyengine_api.fastapi_routes.v2.user_policies.routes import ( - user_policy_error_response, -) from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, @@ -120,26 +117,15 @@ async def typed_v2_request_validation_error( error: RequestValidationError, ) -> Response: if request.url.path.startswith("/v2/"): - if request.url.path.startswith("/v2/user-policies"): - return user_policy_error_response( - 422, - "Invalid v2 user-policy request", - ) - if request.url.path.startswith("/v2/policies"): - return policy_error_response(422, "Invalid v2 policy request") - from policyengine_api.fastapi_routes.v2.metadata.common import ( - error_response, - ) - - return error_response(422, "Invalid v2 metadata request") + return v2_error_response(422, "Invalid API v2 request") return await request_validation_exception_handler(request, error) - @app.exception_handler(PolicyRequestTooLargeError) - async def oversized_policy_request( + @app.exception_handler(V2RequestTooLargeError) + async def oversized_v2_request( _request: Request, - error: PolicyRequestTooLargeError, + error: V2RequestTooLargeError, ) -> Response: - return policy_error_response(413, str(error)) + return v2_error_response(413, str(error)) @app.middleware("http") async def add_cors_for_native_routes(request, call_next): diff --git a/policyengine_api/fastapi_routes/v2/errors.py b/policyengine_api/fastapi_routes/v2/errors.py new file mode 100644 index 000000000..3f69cb9f6 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/errors.py @@ -0,0 +1,29 @@ +"""Shared error response contract for all native API v2 routes.""" + +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, StringConstraints +from starlette.responses import JSONResponse + + +class V2RequestTooLargeError(ValueError): + """Raised when an API v2 request exceeds its documented byte limit.""" + + +class V2ErrorResponse(BaseModel): + """Strict error envelope shared by every API v2 resource.""" + + model_config = ConfigDict(extra="forbid") + + status: Literal["error"] = "error" + message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] + + +def v2_error_response(status_code: int, message: str) -> JSONResponse: + """Serialize one API v2 error without resource-path dispatch.""" + + error = V2ErrorResponse(message=message) + return JSONResponse( + status_code=status_code, + content=error.model_dump(mode="json"), + ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 177f217f2..fefc8b0a9 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -1,4 +1,4 @@ -"""Shared response handling for API v2 metadata routes.""" +"""Execute API v2 metadata reads and translate failures to HTTP responses.""" from __future__ import annotations @@ -18,8 +18,9 @@ InvalidMetadataPageError, MetadataResourceNotFoundError, ) -from policyengine_api.fastapi_routes.v2.metadata.response_models import ( - MetadataErrorResponse, +from policyengine_api.fastapi_routes.v2.errors import ( + V2ErrorResponse, + v2_error_response, ) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes.dependencies import ( @@ -30,40 +31,32 @@ ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The resource request or PolicyEngine.py version is invalid.", }, 404: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The requested catalog resource is absent.", }, 405: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The dormant v2 metadata resources support GET only.", }, 422: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The request parameters do not match the resource schema.", }, 500: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The resource query failed internally.", }, 503: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The initialized v2 catalog is unavailable.", }, } -def error_response(status_code: int, message: str) -> JSONResponse: - error = MetadataErrorResponse(message=message) - return JSONResponse( - status_code=status_code, - content=error.model_dump(mode="json"), - ) - - ResponseT = TypeVar("ResponseT", bound=BaseModel) @@ -84,18 +77,18 @@ def read_resource( reader = factory() return response_type(result=operation(reader)) except (InvalidMetadataPageError, InvalidPolicyEngineVersionError) as error: - return error_response(400, str(error)) + return v2_error_response(400, str(error)) except UnsupportedPreviewCountryError as error: - return error_response(400, f"Unsupported country: {error}") + return v2_error_response(400, f"Unsupported country: {error}") except ( MetadataCatalogVersionNotFoundError, MetadataResourceNotFoundError, ) as error: - return error_response(404, str(error)) + return v2_error_response(404, str(error)) except (V2ConfigurationError, MetadataCatalogUnavailableError): - return error_response(503, "V2 metadata catalog is unavailable") + return v2_error_response(503, "V2 metadata catalog is unavailable") except Exception: # noqa: BLE001 - preview must return typed errors - return error_response(500, "V2 metadata query failed") + return v2_error_response(500, "V2 metadata query failed") finally: if reader is not None: try: diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index 4be86385b..8d6d965a9 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -2,9 +2,7 @@ from __future__ import annotations -from typing import Annotated, Generic, Literal, TypeVar - -from pydantic import StringConstraints +from typing import Generic, Literal, TypeVar from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, @@ -134,8 +132,3 @@ class MetadataEconomyOptionsResponse( MetadataResourceSuccessResponse[MetadataEconomyOptionsResult] ): pass - - -class MetadataErrorResponse(StrictResponseModel): - status: Literal["error"] = "error" - message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] diff --git a/policyengine_api/fastapi_routes/v2/policies/response_models.py b/policyengine_api/fastapi_routes/v2/policies/response_models.py index 0d3173ac4..94784c831 100644 --- a/policyengine_api/fastapi_routes/v2/policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/policies/response_models.py @@ -3,11 +3,12 @@ from __future__ import annotations from datetime import datetime -from typing import Annotated, Any, Generic, Literal, TypeVar +from typing import Any, Generic, Literal, TypeVar from uuid import UUID -from pydantic import BaseModel, ConfigDict, JsonValue, StringConstraints +from pydantic import BaseModel, ConfigDict, JsonValue +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse from policyengine_api.services.v2.policies.types import PolicyPage, PolicyRead @@ -80,38 +81,33 @@ class PolicyPageResponse(PolicySuccessResponse[PolicyPageResult]): pass -class PolicyErrorResponse(StrictPolicyAPIModel): - status: Literal["error"] = "error" - message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] - - POLICY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { - "model": PolicyErrorResponse, + "model": V2ErrorResponse, "description": "The policy content or country selection is invalid.", }, 404: { - "model": PolicyErrorResponse, + "model": V2ErrorResponse, "description": "The selected policy or catalog does not exist.", }, 409: { - "model": PolicyErrorResponse, + "model": V2ErrorResponse, "description": "Immutable policy identity conflicts with stored state.", }, 413: { - "model": PolicyErrorResponse, + "model": V2ErrorResponse, "description": "The policy request body exceeds 1 MiB.", }, 422: { - "model": PolicyErrorResponse, + "model": V2ErrorResponse, "description": "The request does not match the policy schema.", }, 500: { - "model": PolicyErrorResponse, + "model": V2ErrorResponse, "description": "Stored policy integrity validation failed.", }, 503: { - "model": PolicyErrorResponse, + "model": V2ErrorResponse, "description": "Supabase policy persistence is unavailable.", }, } diff --git a/policyengine_api/fastapi_routes/v2/policies/routes.py b/policyengine_api/fastapi_routes/v2/policies/routes.py index dae1a63da..8e2a5d754 100644 --- a/policyengine_api/fastapi_routes/v2/policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/policies/routes.py @@ -22,11 +22,14 @@ POLICY_ERROR_RESPONSES, PolicyDetailResponse, PolicyDetailResult, - PolicyErrorResponse, PolicyItem, PolicyPageResponse, PolicyPageResult, ) +from policyengine_api.fastapi_routes.v2.errors import ( + V2RequestTooLargeError, + v2_error_response, +) from policyengine_api.services.v2.policies.types import NativePolicyCreationInput from policyengine_api.services.v2.policies.validators import ( PolicyCatalogValidationError, @@ -47,10 +50,6 @@ ) -class PolicyRequestTooLargeError(ValueError): - """Raised before persistence when a native policy body exceeds 1 MiB.""" - - async def enforce_policy_request_size(request: Request) -> None: """Bound both declared and actual request bytes before service creation.""" @@ -59,21 +58,13 @@ async def enforce_policy_request_size(request: Request) -> None: try: declared_length = int(content_length) except ValueError as error: - raise PolicyRequestTooLargeError( + raise V2RequestTooLargeError( "Policy request Content-Length is invalid" ) from error if declared_length > MAXIMUM_POLICY_REQUEST_BYTES: - raise PolicyRequestTooLargeError("Policy request body exceeds 1 MiB") + raise V2RequestTooLargeError("Policy request body exceeds 1 MiB") if len(await request.body()) > MAXIMUM_POLICY_REQUEST_BYTES: - raise PolicyRequestTooLargeError("Policy request body exceeds 1 MiB") - - -def policy_error_response(status_code: int, message: str) -> JSONResponse: - error = PolicyErrorResponse(message=message) - return JSONResponse( - status_code=status_code, - content=error.model_dump(mode="json"), - ) + raise V2RequestTooLargeError("Policy request body exceeds 1 MiB") def _service_factory( @@ -97,17 +88,17 @@ def _policy_operation( try: return operation() except PolicyCatalogValidationError as error: - return policy_error_response(400, str(error)) + return v2_error_response(400, str(error)) except (MetadataCatalogVersionNotFoundError, PolicyNotFoundError) as error: - return policy_error_response(404, str(error)) + return v2_error_response(404, str(error)) except PolicyContentHashCollisionError: - return policy_error_response(409, "Policy content hash conflicts with storage") + return v2_error_response(409, "Policy content hash conflicts with storage") except PolicyCreationIntegrityError: - return policy_error_response(500, "Stored policy integrity failed") + return v2_error_response(500, "Stored policy integrity failed") except (V2ConfigurationError, MetadataCatalogUnavailableError, SQLAlchemyError): - return policy_error_response(503, "V2 policy persistence is unavailable") + return v2_error_response(503, "V2 policy persistence is unavailable") except Exception: # noqa: BLE001 - route must return a secret-safe typed error - return policy_error_response(500, "V2 policy operation failed") + return v2_error_response(500, "V2 policy operation failed") def build_v2_policy_router( @@ -139,7 +130,7 @@ def create_policy( _size: None = Depends(enforce_policy_request_size), ) -> PolicyDetailResponse | JSONResponse: if body.country_id != query.country_id: - return policy_error_response( + return v2_error_response( 400, "Body country_id must match query country_id", ) diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index aa54f150c..68bc30fe0 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -5,10 +5,8 @@ from fastapi import APIRouter, Request from starlette.responses import JSONResponse -from policyengine_api.fastapi_routes.v2.metadata.response_models import ( - MetadataErrorResponse, -) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse from policyengine_api.fastapi_routes.v2.metadata.geography_routes import ( build_v2_metadata_geography_router, ) @@ -55,44 +53,44 @@ def v2_preview_openapi(request: Request) -> JSONResponse: @router.get( "/v2", - response_model=MetadataErrorResponse, + response_model=V2ErrorResponse, status_code=404, include_in_schema=False, ) - def unsupported_v2_root() -> MetadataErrorResponse: - return MetadataErrorResponse(message="V2 metadata resource was not found") + def unsupported_v2_root() -> V2ErrorResponse: + return V2ErrorResponse(message="API v2 resource was not found") @router.api_route( "/v2", methods=["POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"], - response_model=MetadataErrorResponse, + response_model=V2ErrorResponse, status_code=405, include_in_schema=False, ) - def unsupported_v2_root_method() -> MetadataErrorResponse: - return MetadataErrorResponse(message="V2 metadata resources support GET only") + def unsupported_v2_root_method() -> V2ErrorResponse: + return V2ErrorResponse(message="API v2 resource does not support this method") @router.get( "/v2/{resource_path:path}", - response_model=MetadataErrorResponse, + response_model=V2ErrorResponse, status_code=404, include_in_schema=False, ) - def unsupported_resource(resource_path: str) -> MetadataErrorResponse: - return MetadataErrorResponse( - message=f"V2 metadata resource {resource_path!r} was not found" + def unsupported_resource(resource_path: str) -> V2ErrorResponse: + return V2ErrorResponse( + message=f"API v2 resource {resource_path!r} was not found" ) @router.api_route( "/v2/{resource_path:path}", methods=["POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"], - response_model=MetadataErrorResponse, + response_model=V2ErrorResponse, status_code=405, include_in_schema=False, ) - def unsupported_method(resource_path: str) -> MetadataErrorResponse: - return MetadataErrorResponse( - message=f"V2 metadata resource {resource_path!r} supports GET only" + def unsupported_method(resource_path: str) -> V2ErrorResponse: + return V2ErrorResponse( + message=f"API v2 resource {resource_path!r} does not support this method" ) return router diff --git a/policyengine_api/fastapi_routes/v2/user_policies/response_models.py b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py index 110a18e39..caef44df3 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py @@ -3,11 +3,12 @@ from __future__ import annotations from datetime import datetime -from typing import Annotated, Any, Generic, Literal, TypeVar +from typing import Any, Generic, Literal, TypeVar from uuid import UUID -from pydantic import BaseModel, ConfigDict, StringConstraints +from pydantic import BaseModel, ConfigDict +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse from policyengine_api.services.v2.user_policies.types import ( UserPolicyPage, UserPolicyRead, @@ -73,34 +74,29 @@ class UserPolicyPageResponse(UserPolicySuccessResponse[UserPolicyPageResult]): pass -class UserPolicyErrorResponse(StrictUserPolicyAPIModel): - status: Literal["error"] = "error" - message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] - - USER_POLICY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { - "model": UserPolicyErrorResponse, + "model": V2ErrorResponse, "description": "Association content or country selection is invalid.", }, 404: { - "model": UserPolicyErrorResponse, + "model": V2ErrorResponse, "description": "The selected user, policy, or association does not exist.", }, 409: { - "model": UserPolicyErrorResponse, + "model": V2ErrorResponse, "description": "Association state conflicts with stored state.", }, 422: { - "model": UserPolicyErrorResponse, + "model": V2ErrorResponse, "description": "The request does not match the association schema.", }, 500: { - "model": UserPolicyErrorResponse, + "model": V2ErrorResponse, "description": "The association operation could not be completed.", }, 503: { - "model": UserPolicyErrorResponse, + "model": V2ErrorResponse, "description": "Supabase association persistence is unavailable.", }, } diff --git a/policyengine_api/fastapi_routes/v2/user_policies/routes.py b/policyengine_api/fastapi_routes/v2/user_policies/routes.py index 620ee2703..7b4f78c77 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/routes.py @@ -19,11 +19,11 @@ USER_POLICY_ERROR_RESPONSES, UserPolicyDetailResponse, UserPolicyDetailResult, - UserPolicyErrorResponse, UserPolicyItem, UserPolicyPageResponse, UserPolicyPageResult, ) +from policyengine_api.fastapi_routes.v2.errors import v2_error_response from policyengine_api.services.v2.user_policies.validators import ( UserPolicyNotFoundError, AssociationCountryConflictError, @@ -41,14 +41,6 @@ ) -def user_policy_error_response(status_code: int, message: str) -> JSONResponse: - error = UserPolicyErrorResponse(message=message) - return JSONResponse( - status_code=status_code, - content=error.model_dump(mode="json"), - ) - - def _service_factory( dependencies: NativeRouteDependencies, ) -> Callable[[], V2UserPolicyResourceService]: @@ -70,20 +62,20 @@ def _association_operation( try: return operation() except AssociationCountryConflictError as error: - return user_policy_error_response(400, str(error)) + return v2_error_response(400, str(error)) except ( AssociationPolicyNotFoundError, AssociationUserNotFoundError, UserPolicyNotFoundError, ) as error: - return user_policy_error_response(404, str(error)) + return v2_error_response(404, str(error)) except (V2ConfigurationError, SQLAlchemyError): - return user_policy_error_response( + return v2_error_response( 503, "V2 association persistence is unavailable", ) except Exception: # noqa: BLE001 - return a secret-safe typed error - return user_policy_error_response(500, "V2 association operation failed") + return v2_error_response(500, "V2 association operation failed") def build_v2_user_policy_router( @@ -113,7 +105,7 @@ def create_user_policy( query: CountryQuery = Depends(country_query), ) -> UserPolicyDetailResponse | JSONResponse: if body.country_id != query.country_id: - return user_policy_error_response( + return v2_error_response( 400, "Body country_id must match query country_id", ) diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 59ab37914..466745e17 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -48,7 +48,7 @@ def test_live_v2_openapi_describes_every_preview_resource(api_client) -> None: expected_paths.update({"/v2/parameters/children", "/v2/economy-options"}) assert expected_paths <= document["paths"].keys() assert all("get" in document["paths"][path] for path in expected_paths) - assert "MetadataErrorResponse" in document["components"]["schemas"] + assert "V2ErrorResponse" in document["components"]["schemas"] @pytest.mark.parametrize("country_id", ["us", "uk"]) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index aae650bd8..afbf54d56 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -372,7 +372,7 @@ def factory(): assert response.status_code == 422 assert response.json() == { "status": "error", - "message": "Invalid v2 metadata request", + "message": "Invalid API v2 request", } assert calls == [] @@ -544,6 +544,12 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/user-policies/{association_id}", } assert set(schema["paths"]) == metadata_paths | native_paths + assert "V2ErrorResponse" in schema["components"]["schemas"] + assert { + "MetadataErrorResponse", + "PolicyErrorResponse", + "UserPolicyErrorResponse", + }.isdisjoint(schema["components"]["schemas"]) for path in metadata_paths: operation = schema["paths"][path]["get"] @@ -564,4 +570,4 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: error_schema = operation["responses"][status]["content"][ "application/json" ]["schema"] - assert error_schema["$ref"] == "#/components/schemas/MetadataErrorResponse" + assert error_schema["$ref"] == "#/components/schemas/V2ErrorResponse" diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py index ebbe81863..900de1abd 100644 --- a/tests/unit/v2/test_policy_routes.py +++ b/tests/unit/v2/test_policy_routes.py @@ -202,6 +202,10 @@ def test_create_rejects_country_mismatch_and_core_presentation_fields() -> None: assert mismatch.status_code == 400 assert named.status_code == 422 + assert named.json() == { + "status": "error", + "message": "Invalid API v2 request", + } assert service.calls == [] @@ -237,6 +241,10 @@ def test_create_rejects_unknown_duplicate_and_oversized_input_before_service() - ) assert unknown.status_code == 422 + assert unknown.json() == { + "status": "error", + "message": "Invalid API v2 request", + } assert duplicate.status_code == 422 assert excessive_values.status_code == 422 assert oversized.status_code == 413 diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py index 8d12d6ca9..922a99e08 100644 --- a/tests/unit/v2/test_user_policy_routes.py +++ b/tests/unit/v2/test_user_policy_routes.py @@ -216,6 +216,10 @@ def test_create_rejects_country_mismatch_and_invalid_fields() -> None: assert mismatch.status_code == 400 assert invalid_user.status_code == 422 + assert invalid_user.json() == { + "status": "error", + "message": "Invalid API v2 request", + } assert long_name.status_code == 422 assert service.calls == [] From d67bff3d7e53b888ec9c5220def09d7acbc05507 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:20:28 +0400 Subject: [PATCH 14/18] Fix Phase 10 mirror selection and coverage --- .github/workflows/v2-integration-check.yml | 2 ++ docs/engineering/migration-contracts.md | 2 +- docs/generated/migration_contracts.json | 2 +- policyengine_api/routes/policy_routes.py | 1 + policyengine_api/services/policy_service.py | 24 +++++++++------ .../services/user_policy_service.py | 18 +++++++----- tests/contract/registry.py | 2 +- .../integration/test_v1_policy_dual_write.py | 1 + .../test_v1_user_policy_dual_write.py | 2 ++ .../routes/test_policy_dual_write_routes.py | 10 +++++-- tests/unit/services/test_policy_service.py | 21 ++++++++++++++ .../unit/services/test_user_policy_service.py | 29 ++++++++++++++++--- tests/unit/test_alembic_workflows.py | 6 +++- 13 files changed, 93 insertions(+), 27 deletions(-) diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index b69399a7b..40e362171 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -57,6 +57,8 @@ jobs: RUN_V2_CATALOG_COMPATIBILITY: "1" - name: Test v2 metadata publication and resource routes run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py + - name: Test v2 policy persistence and immediate v1 mirroring + run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Qualify production-scale v2 metadata publication run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication_qualification.py env: diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index 7e615346f..92691c1c1 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -70,7 +70,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | --- | ---: | --- | --- | | `POST` | `/v2/user-policies?country_id=us` | 201 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.user_id`, `result.item.policy_id`, `result.item.name`, `result.item.description`, `result.item.created_at`, `result.item.updated_at` | | `GET` | `/v2/user-policies/{association_id}?country_id=us` | 200 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.user_id`, `result.item.policy_id`, `result.item.name`, `result.item.description`, `result.item.created_at`, `result.item.updated_at` | -| `GET` | `/v2/user-policies?country_id=us&user_id=caller` | 200 | `policy` | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | +| `GET` | `/v2/user-policies?country_id=us&user_id={user_id}` | 200 | `policy` | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | | `PATCH` | `/v2/user-policies/{association_id}?country_id=us` | 200 | `policy` | `status`, `message`, `result.item.id`, `result.item.name`, `result.item.description`, `result.item.updated_at` | | `DELETE` | `/v2/user-policies/{association_id}?country_id=us` | 204 | `policy` | | diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 8b1d026f5..bf976fccb 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -278,7 +278,7 @@ { "expected_status": 200, "method": "GET", - "path": "/v2/user-policies?country_id=us&user_id=caller", + "path": "/v2/user-policies?country_id=us&user_id={user_id}", "route_group": "policy", "stable_response_fields": [ "status", diff --git a/policyengine_api/routes/policy_routes.py b/policyengine_api/routes/policy_routes.py index e3bfe3a7a..9841f8d70 100644 --- a/policyengine_api/routes/policy_routes.py +++ b/policyengine_api/routes/policy_routes.py @@ -196,6 +196,7 @@ def set_policy(country_id: str) -> Response: country_id, label, policy_json, + prepare_for_mirroring=write_source == "dual_write", ) policy_id, message, is_existing_policy = creation diff --git a/policyengine_api/services/policy_service.py b/policyengine_api/services/policy_service.py index 32c7a8e47..b8b011f43 100644 --- a/policyengine_api/services/policy_service.py +++ b/policyengine_api/services/policy_service.py @@ -17,12 +17,12 @@ @dataclass(frozen=True) class PolicySetResult: - """Existing v1 return values plus a detached committed-row snapshot.""" + """Existing v1 return values and an optional detached mirror snapshot.""" policy_id: int message: str is_existing_policy: bool - snapshot: LegacyPolicySnapshot + snapshot: LegacyPolicySnapshot | None def __iter__(self) -> Iterator[int | str | bool]: """Preserve the established three-value internal unpacking interface.""" @@ -136,6 +136,8 @@ def set_policy( country_id: str, label: str | None, policy_json: dict, + *, + prepare_for_mirroring: bool = False, ) -> PolicySetResult: country_id = country_id.lower() if country_id not in COUNTRY_PACKAGE_VERSIONS: @@ -150,13 +152,17 @@ def set_policy( policy_json, policy_hash, ) - snapshot = LegacyPolicySnapshot( - country_id=policy.country_id, - legacy_policy_id=policy.id, - label=policy.label, - api_version=policy.api_version, - policy_json=copy.deepcopy(policy.policy_json), - source_policy_hash=policy.policy_hash, + snapshot = ( + LegacyPolicySnapshot( + country_id=policy.country_id, + legacy_policy_id=policy.id, + label=policy.label, + api_version=policy.api_version, + policy_json=copy.deepcopy(policy.policy_json), + source_policy_hash=policy.policy_hash, + ) + if prepare_for_mirroring + else None ) return PolicySetResult( policy_id=policy.id, diff --git a/policyengine_api/services/user_policy_service.py b/policyengine_api/services/user_policy_service.py index 60904dd85..ab0c4e7d3 100644 --- a/policyengine_api/services/user_policy_service.py +++ b/policyengine_api/services/user_policy_service.py @@ -108,14 +108,14 @@ class PendingUserPolicyMirrorEvent: class UserPolicyCreateResult: user_policy: UserPolicy created: bool - snapshot: LegacyUserPolicySnapshot + snapshot: LegacyUserPolicySnapshot | None mirror_revision: int | None = None @dataclass(frozen=True) class UserPolicyUpdateResult: user_policy: UserPolicy - snapshot: LegacyUserPolicySnapshot + snapshot: LegacyUserPolicySnapshot | None changed_fields: frozenset[str] mirror_revision: int | None = None @@ -179,7 +179,7 @@ def _record_mirror_event( *, event_type: str, changed_fields: frozenset[str], - ) -> int: + ) -> tuple[int, LegacyUserPolicySnapshot]: user_policy.mirror_revision += 1 snapshot = cls._snapshot(user_policy) event = UserPolicyMirrorEvent( @@ -196,7 +196,7 @@ def _record_mirror_event( ) session.add_all((user_policy, event)) session.flush() - return user_policy.mirror_revision + return user_policy.mirror_revision, snapshot @staticmethod def _decode_mirror_event( @@ -286,8 +286,9 @@ def create_or_get_user_policy( session.add(user_policy) session.flush() mirror_revision = None + snapshot = None if record_mirror_event: - mirror_revision = self._record_mirror_event( + mirror_revision, snapshot = self._record_mirror_event( session, user_policy, event_type="create", @@ -296,7 +297,7 @@ def create_or_get_user_policy( result = UserPolicyCreateResult( user_policy=user_policy, created=created, - snapshot=self._snapshot(user_policy), + snapshot=snapshot, mirror_revision=mirror_revision, ) except UserPolicyPersistenceError: @@ -344,8 +345,9 @@ def update_user_policy( session.flush() changed_fields = frozenset(values) mirror_revision = None + snapshot = None if record_mirror_event: - mirror_revision = self._record_mirror_event( + mirror_revision, snapshot = self._record_mirror_event( session, user_policy, event_type="update", @@ -353,7 +355,7 @@ def update_user_policy( ) result = UserPolicyUpdateResult( user_policy=user_policy, - snapshot=self._snapshot(user_policy), + snapshot=snapshot, changed_fields=changed_fields, mirror_revision=mirror_revision, ) diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 92dcb494d..0c5c26df8 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -185,7 +185,7 @@ class WorkflowContract: ), ContractRequest( method="GET", - path="/v2/user-policies?country_id=us&user_id=caller", + path="/v2/user-policies?country_id=us&user_id={user_id}", expected_status=200, stable_response_fields=( "status", diff --git a/tests/integration/test_v1_policy_dual_write.py b/tests/integration/test_v1_policy_dual_write.py index a8f4381ae..69f272605 100644 --- a/tests/integration/test_v1_policy_dual_write.py +++ b/tests/integration/test_v1_policy_dual_write.py @@ -133,6 +133,7 @@ def _create_v1(service: PolicyService, parameter_name: str): "us", "Cross-database policy", {parameter_name: {"2026": 0.2}}, + prepare_for_mirroring=True, ) diff --git a/tests/integration/test_v1_user_policy_dual_write.py b/tests/integration/test_v1_user_policy_dual_write.py index a57819f52..63383f27b 100644 --- a/tests/integration/test_v1_user_policy_dual_write.py +++ b/tests/integration/test_v1_user_policy_dual_write.py @@ -214,6 +214,7 @@ def test_create_update_and_v1_only_change_mirror_one_association() -> None: "us", "Legacy reform label", {parameter_name: {"2026": 0.2}}, + prepare_for_mirroring=True, ) created = saved_service.create_or_get_user_policy( _saved_values(reform.policy_id), @@ -377,6 +378,7 @@ def test_destination_commit_replays_when_source_processing_marker_is_missing() - "us", "Legacy reform label", {parameter_name: {"2026": 0.2}}, + prepare_for_mirroring=True, ) created = saved_service.create_or_get_user_policy( _saved_values(reform.policy_id), diff --git a/tests/unit/routes/test_policy_dual_write_routes.py b/tests/unit/routes/test_policy_dual_write_routes.py index 762a95a13..389478b4d 100644 --- a/tests/unit/routes/test_policy_dual_write_routes.py +++ b/tests/unit/routes/test_policy_dual_write_routes.py @@ -69,7 +69,12 @@ def test_cloud_sql_mode_preserves_v1_create_response_without_mirroring( "message": "Policy created", "result": {"policy_id": 42}, } - set_policy.assert_called_once() + set_policy.assert_called_once_with( + "us", + "Legacy label", + {"gov.example.rate": {"2026": 0.2}}, + prepare_for_mirroring=False, + ) mirror.assert_not_called() @@ -80,7 +85,7 @@ def test_dual_write_mirrors_new_and_existing_rows_before_success( for existing, expected_status in ((False, 201), (True, 200)): events: list[str] = [] service = MagicMock( - side_effect=lambda *_args: ( + side_effect=lambda *_args, **_kwargs: ( events.append("cloud_sql") or _creation(existing=existing) ) ) @@ -101,6 +106,7 @@ def test_dual_write_mirrors_new_and_existing_rows_before_success( assert response.json["result"] == {"policy_id": 42} assert "v2" not in response.json["result"] assert events == ["cloud_sql", "supabase"] + assert service.call_args.kwargs == {"prepare_for_mirroring": True} mirror.assert_called_once_with(_snapshot()) diff --git a/tests/unit/services/test_policy_service.py b/tests/unit/services/test_policy_service.py index 121dbe28e..c8f008dd7 100644 --- a/tests/unit/services/test_policy_service.py +++ b/tests/unit/services/test_policy_service.py @@ -103,6 +103,7 @@ def test_set_policy_adds_mapped_entity(service, monkeypatch): "US", "New policy", {"parameter": 1}, + prepare_for_mirroring=True, ) policy_id, message, exists = result @@ -136,6 +137,7 @@ def test_set_policy_returns_existing_mapped_entity( "us", None, {}, + prepare_for_mirroring=True, ) policy_id, message, exists = result @@ -146,6 +148,25 @@ def test_set_policy_returns_existing_mapped_entity( assert result.snapshot.source_policy_hash == valid_policy_data["policy_hash"] +def test_set_policy_does_not_build_v2_snapshot_unless_requested( + service, + monkeypatch, +): + monkeypatch.setattr( + "policyengine_api.services.policy_service.hash_object", + lambda value: "new-hash", + ) + + result = service.set_policy( + "ca", + "Canadian policy", + {"parameter": 1}, + ) + + assert result.snapshot is None + assert service.get_policy("ca", result.policy_id) is not None + + def test_set_policy_rejects_invalid_country(service): with pytest.raises(ValueError, match="Invalid country_id: xx"): service.set_policy("xx", "Policy", {}) diff --git a/tests/unit/services/test_user_policy_service.py b/tests/unit/services/test_user_policy_service.py index 51d361138..68f275ffa 100644 --- a/tests/unit/services/test_user_policy_service.py +++ b/tests/unit/services/test_user_policy_service.py @@ -165,18 +165,34 @@ def test_create_reuse_list_and_update_user_policy(orm_session_factory): assert created.created is True assert reused.created is False assert reused.user_policy.id == created.user_policy.id - assert reused.snapshot == created.snapshot + assert created.snapshot is None + assert reused.snapshot is None assert len(listed) == 1 assert isinstance(listed[0], UserPolicy) assert isinstance(updated, UserPolicyUpdateResult) assert updated.user_policy.reform_label == "Updated" assert updated.user_policy.updated_date == 3 - assert updated.snapshot.reform_label == "Updated" - assert updated.snapshot.updated_date == 3 - assert updated.snapshot.legacy_user_policy_id == created.user_policy.id + assert updated.snapshot is None assert updated.changed_fields == frozenset({"reform_label", "updated_date"}) +def test_v1_only_saved_policy_mutations_do_not_build_v2_snapshots( + orm_session_factory, +): + service = UserPolicyService(orm_session_factory) + + created = service.create_or_get_user_policy(_values(country_id="ca")) + updated = service.update_user_policy( + "ca", + created.user_policy.id, + {"reform_label": "Canadian policy"}, + ) + + assert created.snapshot is None + assert updated is not None + assert updated.snapshot is None + + def test_update_user_policy_requires_matching_country(orm_session_factory): service = UserPolicyService(orm_session_factory) created = service.create_or_get_user_policy(_values(country_id="uk")) @@ -210,8 +226,13 @@ def test_dual_write_mutations_store_ordered_complete_events_atomically( ) assert created.mirror_revision == 1 + assert created.snapshot is not None assert updated is not None assert updated.mirror_revision == 2 + assert updated.snapshot is not None + assert updated.snapshot.reform_label is None + assert updated.snapshot.updated_date == 3 + assert updated.snapshot.legacy_user_policy_id == created.user_policy.id with orm_session_factory() as session: stored = session.get(UserPolicy, created.user_policy.id) events = session.scalars( diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index b996c79f1..6e0409686 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -180,12 +180,16 @@ def test_reusable_v2_integration_check_uses_postgres_redis_and_coverage(): assert "RUN_V2_CATALOG_COMPATIBILITY" in workflow assert "test_v2_catalog_publication.py" in workflow assert "test_v2_metadata_routes.py" in workflow + assert "test_v2_policy_persistence.py" in workflow + assert "test_v1_policy_dual_write.py" in workflow + assert "test_v2_user_policy_mirroring.py" in workflow + assert "test_v1_user_policy_dual_write.py" in workflow assert "test_v2_catalog_publication_qualification.py" in workflow assert "RUN_V2_CATALOG_PUBLICATION_QUALIFICATION" in workflow assert "test_runtime_cache_redis.py" in workflow assert "uv sync --frozen" in workflow assert workflow.count("coverage run --branch") == 1 - assert workflow.count("coverage run -a --branch") == 3 + assert workflow.count("coverage run -a --branch") == 4 assert "coverage xml -i -o coverage-v2.xml" in workflow assert "codecov/codecov-action@v5" in workflow assert "files: coverage-v2.xml" in workflow From 231e6e5c2955720d0919b0aec71e26f497d6404f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:10:33 +0400 Subject: [PATCH 15/18] Scope Phase 10 mirroring to v2 countries --- changelog.d/stage-10.added.md | 2 +- docs/migration/stage-10-v2-policies.md | 11 +- .../data/v2/catalog/catalog_selection.py | 4 +- policyengine_api/routes/policy_routes.py | 28 +++-- .../routes/test_policy_dual_write_routes.py | 56 +++++++-- .../test_user_policy_dual_write_routes.py | 113 ++++++++++++++++-- .../unit/services/test_user_policy_service.py | 2 + 7 files changed, 182 insertions(+), 34 deletions(-) diff --git a/changelog.d/stage-10.added.md b/changelog.d/stage-10.added.md index 50acd514a..35cdf955f 100644 --- a/changelog.d/stage-10.added.md +++ b/changelog.d/stage-10.added.md @@ -1 +1 @@ -Add API v2 policies and user-policy associations with immediate API v1 mutation mirroring. +Add US and UK API v2 policies and user-policy associations with immediate API v1 mutation mirroring while other v1-supported countries remain Cloud SQL-only. diff --git a/docs/migration/stage-10-v2-policies.md b/docs/migration/stage-10-v2-policies.md index 134ec7af3..e9f4232f2 100644 --- a/docs/migration/stage-10-v2-policies.md +++ b/docs/migration/stage-10-v2-policies.md @@ -6,7 +6,7 @@ integer identifiers remain in Cloud SQL throughout this stage. ## Preconditions -1. Stage 9 metadata catalogs must be initialized for every supported country +1. Stage 9 metadata catalogs must be initialized for the v2-supported US and UK and the running PolicyEngine.py version. Policy creation does not fall back to v1 metadata or another catalog version. 2. The runtime Supabase URL and target identity settings must identify the same @@ -74,8 +74,8 @@ DB_READ_POLICY=cloud_sql DB_WRITE_POLICY=cloud_sql ``` -After native lifecycle checks pass, require immediate mirroring for v1 policy -and saved-policy mutations: +After native lifecycle checks pass, require immediate mirroring for US and UK +v1 policy and saved-policy mutations: ```text DB_READ_POLICY=cloud_sql @@ -89,6 +89,11 @@ transaction. The same request then processes that source's pending events in revision order and records `processed_at` only after the corresponding Supabase transaction commits. +Canada, Nigeria, and Israel remain supported by v1 but have no v2 catalog in +this phase. Their policy and saved-policy mutations continue using Cloud SQL +only under `DB_WRITE_POLICY=dual_write`: they create no v2 snapshot or mirror +event and do not access Supabase. + A Supabase failure returns HTTP 503. An identical client retry reads the already committed v1 row, appends the next revision when applicable, and first replays any retained earlier event. Destination revision and fingerprint diff --git a/policyengine_api/data/v2/catalog/catalog_selection.py b/policyengine_api/data/v2/catalog/catalog_selection.py index 7014210c9..e4a32fb3c 100644 --- a/policyengine_api/data/v2/catalog/catalog_selection.py +++ b/policyengine_api/data/v2/catalog/catalog_selection.py @@ -14,7 +14,7 @@ ) -SUPPORTED_PREVIEW_COUNTRIES = frozenset({"us", "uk"}) +SUPPORTED_V2_COUNTRY_IDS = frozenset({"us", "uk"}) class MetadataCatalogUnavailableError(RuntimeError): @@ -76,7 +76,7 @@ def select_catalog( ) -> SelectedCatalog: """Select one country catalog using the requested or running package version.""" - if country_id not in SUPPORTED_PREVIEW_COUNTRIES: + if country_id not in SUPPORTED_V2_COUNTRY_IDS: raise UnsupportedPreviewCountryError(country_id) explicit_version = policyengine_version is not None selected_version = ( diff --git a/policyengine_api/routes/policy_routes.py b/policyengine_api/routes/policy_routes.py index 9841f8d70..3918703e7 100644 --- a/policyengine_api/routes/policy_routes.py +++ b/policyengine_api/routes/policy_routes.py @@ -5,6 +5,9 @@ from werkzeug.exceptions import BadRequest, NotFound from policyengine_api.data.v1_models import Policy, UserPolicy +from policyengine_api.data.v2.catalog.catalog_selection import ( + SUPPORTED_V2_COUNTRY_IDS, +) from policyengine_api.gcp_logging import logger from policyengine_api.migration_flags import ( get_v1_policy_read_source, @@ -36,6 +39,12 @@ user_policy_service = UserPolicyService() +def _should_mirror_to_v2(country_id: str, write_source: str) -> bool: + """Select immediate mirroring only for countries supported by API v2.""" + + return write_source == "dual_write" and country_id in SUPPORTED_V2_COUNTRY_IDS + + def _policy_configuration_unavailable() -> Response: return _make_error_response( "Policy persistence configuration is unavailable.", @@ -191,16 +200,17 @@ def set_policy(country_id: str) -> Response: label = payload.pop("label", None) policy_json = payload.pop("data", None) + mirror_to_v2 = _should_mirror_to_v2(country_id, write_source) creation = policy_service.set_policy( country_id, label, policy_json, - prepare_for_mirroring=write_source == "dual_write", + prepare_for_mirroring=mirror_to_v2, ) policy_id, message, is_existing_policy = creation - if write_source == "dual_write": + if mirror_to_v2: snapshot = getattr(creation, "snapshot", None) if snapshot is None: return _policy_mirror_unavailable() @@ -312,7 +322,8 @@ def set_user_policy(country_id: str) -> Response: try: write_source = get_v1_policy_write_source() - if write_source == "dual_write": + mirror_to_v2 = _should_mirror_to_v2(country_id, write_source) + if mirror_to_v2: get_v1_policy_read_source() except ValueError: return _policy_configuration_unavailable() @@ -321,7 +332,7 @@ def set_user_policy(country_id: str) -> Response: try: creation = user_policy_service.create_or_get_user_policy( values, - record_mirror_event=write_source == "dual_write", + record_mirror_event=mirror_to_v2, ) user_policy = creation.user_policy except UserPolicyPersistenceError as error: @@ -333,7 +344,7 @@ def set_user_policy(country_id: str) -> Response: started_at=persistence_started_at, ) - if write_source == "dual_write": + if mirror_to_v2: if creation.mirror_revision is None: return _user_policy_mirror_unavailable() try: @@ -429,7 +440,8 @@ def update_user_policy(country_id: str) -> Response: try: write_source = get_v1_policy_write_source() - if write_source == "dual_write": + mirror_to_v2 = _should_mirror_to_v2(country_id, write_source) + if mirror_to_v2: get_v1_policy_read_source() except ValueError: return _policy_configuration_unavailable() @@ -440,7 +452,7 @@ def update_user_policy(country_id: str) -> Response: country_id, user_policy_id, payload, - record_mirror_event=write_source == "dual_write", + record_mirror_event=mirror_to_v2, ) except UserPolicyPersistenceError as error: return _user_policy_persistence_failure( @@ -459,7 +471,7 @@ def update_user_policy(country_id: str) -> Response: include_status=False, ) - if write_source == "dual_write": + if mirror_to_v2: if update.mirror_revision is None: return _user_policy_mirror_unavailable() try: diff --git a/tests/unit/routes/test_policy_dual_write_routes.py b/tests/unit/routes/test_policy_dual_write_routes.py index 389478b4d..39b2e1771 100644 --- a/tests/unit/routes/test_policy_dual_write_routes.py +++ b/tests/unit/routes/test_policy_dual_write_routes.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patch from flask import Flask +import pytest from policyengine_api.data.v1_models import Policy from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot @@ -21,9 +22,9 @@ def _client(): return app.test_client() -def _snapshot() -> LegacyPolicySnapshot: +def _snapshot(country_id: str = "us") -> LegacyPolicySnapshot: return LegacyPolicySnapshot( - country_id="us", + country_id=country_id, legacy_policy_id=42, label="Legacy label", api_version="1.0.0", @@ -32,12 +33,16 @@ def _snapshot() -> LegacyPolicySnapshot: ) -def _creation(*, existing: bool = False) -> PolicySetResult: +def _creation( + *, + existing: bool = False, + country_id: str = "us", +) -> PolicySetResult: return PolicySetResult( policy_id=42, message="Policy already exists" if existing else "Policy created", is_existing_policy=existing, - snapshot=_snapshot(), + snapshot=_snapshot(country_id), ) @@ -78,15 +83,18 @@ def test_cloud_sql_mode_preserves_v1_create_response_without_mirroring( mirror.assert_not_called() +@pytest.mark.parametrize("country_id", ("us", "uk")) def test_dual_write_mirrors_new_and_existing_rows_before_success( monkeypatch, + country_id, ) -> None: monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") for existing, expected_status in ((False, 201), (True, 200)): events: list[str] = [] service = MagicMock( side_effect=lambda *_args, **_kwargs: ( - events.append("cloud_sql") or _creation(existing=existing) + events.append("cloud_sql") + or _creation(existing=existing, country_id=country_id) ) ) mirror = MagicMock(side_effect=lambda _snapshot: events.append("supabase")) @@ -100,14 +108,48 @@ def test_dual_write_mirrors_new_and_existing_rows_before_success( mirror, ), ): - response = _client().post("/us/policy", json=_body()) + response = _client().post(f"/{country_id}/policy", json=_body()) assert response.status_code == expected_status assert response.json["result"] == {"policy_id": 42} assert "v2" not in response.json["result"] assert events == ["cloud_sql", "supabase"] assert service.call_args.kwargs == {"prepare_for_mirroring": True} - mirror.assert_called_once_with(_snapshot()) + mirror.assert_called_once_with(_snapshot(country_id)) + + +@pytest.mark.parametrize("country_id", ("ca", "ng", "il")) +def test_dual_write_preserves_v1_only_policy_writes_for_non_v2_countries( + monkeypatch, + country_id, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + creation = PolicySetResult( + policy_id=42, + message="Policy created", + is_existing_policy=False, + snapshot=None, + ) + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy", + return_value=creation, + ) as set_policy, + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit" + ) as mirror, + ): + response = _client().post(f"/{country_id}/policy", json=_body()) + + assert response.status_code == 201 + assert response.json["result"] == {"policy_id": 42} + set_policy.assert_called_once_with( + country_id, + "Legacy label", + {"gov.example.rate": {"2026": 0.2}}, + prepare_for_mirroring=False, + ) + mirror.assert_not_called() def test_mirror_failure_returns_503_and_identical_retry_completes( diff --git a/tests/unit/routes/test_user_policy_dual_write_routes.py b/tests/unit/routes/test_user_policy_dual_write_routes.py index 7f003a8b5..5b9d97734 100644 --- a/tests/unit/routes/test_user_policy_dual_write_routes.py +++ b/tests/unit/routes/test_user_policy_dual_write_routes.py @@ -35,10 +35,15 @@ def _client(): return app.test_client() -def _row(*, reform_label: str | None = "Reform", year: str = "2026"): +def _row( + *, + country_id: str = "us", + reform_label: str | None = "Reform", + year: str = "2026", +): return UserPolicy( id=10, - country_id="us", + country_id=country_id, reform_id=2, reform_label=reform_label, baseline_id=1, @@ -56,9 +61,14 @@ def _row(*, reform_label: str | None = "Reform", year: str = "2026"): ) -def _snapshot(*, reform_label: str | None = "Reform", year: str = "2026"): +def _snapshot( + *, + country_id: str = "us", + reform_label: str | None = "Reform", + year: str = "2026", +): return LegacyUserPolicySnapshot( - country_id="us", + country_id=country_id, legacy_user_policy_id=10, reform_id=2, reform_label=reform_label, @@ -77,9 +87,9 @@ def _snapshot(*, reform_label: str | None = "Reform", year: str = "2026"): ) -def _reform_snapshot(): +def _reform_snapshot(country_id: str = "us"): return LegacyPolicySnapshot( - country_id="us", + country_id=country_id, legacy_policy_id=2, label="Ignored core label", api_version="1.0.0", @@ -88,11 +98,17 @@ def _reform_snapshot(): ) -def _creation(*, created=True, reform_label="Reform", mirror_revision=1): +def _creation( + *, + created=True, + country_id="us", + reform_label="Reform", + mirror_revision=1, +): return UserPolicyCreateResult( - user_policy=_row(reform_label=reform_label), + user_policy=_row(country_id=country_id, reform_label=reform_label), created=created, - snapshot=_snapshot(reform_label=reform_label), + snapshot=_snapshot(country_id=country_id, reform_label=reform_label), mirror_revision=mirror_revision, ) @@ -142,8 +158,10 @@ def test_cloud_sql_mode_preserves_create_without_association_mirror( get_reform.assert_not_called() +@pytest.mark.parametrize("country_id", ("us", "uk")) def test_dual_write_mirrors_new_existing_and_unlabeled_saved_policies( monkeypatch, + country_id, ) -> None: monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") for created, label, expected_status in ( @@ -151,7 +169,11 @@ def test_dual_write_mirrors_new_existing_and_unlabeled_saved_policies( (False, "Reform", 200), (True, None, 201), ): - creation = _creation(created=created, reform_label=label) + creation = _creation( + created=created, + country_id=country_id, + reform_label=label, + ) with ( patch( "policyengine_api.routes.policy_routes.user_policy_service.create_or_get_user_policy", @@ -159,7 +181,7 @@ def test_dual_write_mirrors_new_existing_and_unlabeled_saved_policies( ), patch( "policyengine_api.routes.policy_routes.policy_service.get_policy_snapshot", - return_value=_reform_snapshot(), + return_value=_reform_snapshot(country_id), ), patch( "policyengine_api.routes.policy_routes." @@ -167,7 +189,7 @@ def test_dual_write_mirrors_new_existing_and_unlabeled_saved_policies( ) as mirror, ): response = _client().post( - "/us/user-policy", + f"/{country_id}/user-policy", json=_body(reform_label=label), ) @@ -175,10 +197,75 @@ def test_dual_write_mirrors_new_existing_and_unlabeled_saved_policies( assert response.json["result"]["id"] == 10 assert "v2" not in response.json["result"] mirror.assert_called_once() - assert mirror.call_args.args == ("us", 10) + assert mirror.call_args.args == (country_id, 10) assert mirror.call_args.kwargs["through_revision"] == 1 +@pytest.mark.parametrize("country_id", ("ca", "ng", "il")) +def test_dual_write_preserves_v1_only_saved_policy_mutations_for_non_v2_countries( + monkeypatch, + country_id, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + creation = UserPolicyCreateResult( + user_policy=_row(country_id=country_id), + created=True, + snapshot=None, + mirror_revision=None, + ) + update = UserPolicyUpdateResult( + user_policy=_row(country_id=country_id, year="2027"), + snapshot=None, + changed_fields=frozenset({"year"}), + mirror_revision=None, + ) + + with ( + patch( + "policyengine_api.routes.policy_routes." + "user_policy_service.create_or_get_user_policy", + return_value=creation, + ) as create, + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as create_mirror, + ): + create_response = _client().post( + f"/{country_id}/user-policy", + json=_body(), + ) + + assert create_response.status_code == 201 + assert create.call_args.kwargs == {"record_mirror_event": False} + create_mirror.assert_not_called() + + with ( + patch( + "policyengine_api.routes.policy_routes." + "user_policy_service.update_user_policy", + return_value=update, + ) as update_saved_policy, + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as update_mirror, + ): + update_response = _client().put( + f"/{country_id}/user-policy", + json={"id": 10, "year": "2027"}, + ) + + assert update_response.status_code == 200 + update_saved_policy.assert_called_once_with( + country_id, + 10, + {"year": "2027"}, + record_mirror_event=False, + ) + update_mirror.assert_not_called() + + def test_saved_policy_mirror_failure_returns_503_and_retry_completes( monkeypatch, ) -> None: diff --git a/tests/unit/services/test_user_policy_service.py b/tests/unit/services/test_user_policy_service.py index 68f275ffa..7fec2014e 100644 --- a/tests/unit/services/test_user_policy_service.py +++ b/tests/unit/services/test_user_policy_service.py @@ -191,6 +191,8 @@ def test_v1_only_saved_policy_mutations_do_not_build_v2_snapshots( assert created.snapshot is None assert updated is not None assert updated.snapshot is None + with orm_session_factory() as session: + assert session.scalar(select(func.count(UserPolicyMirrorEvent.id))) == 0 def test_update_user_policy_requires_matching_country(orm_session_factory): From 989ebcc359365438e0dd53e3e8e3a1ffedf7b9a2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:33:41 +0400 Subject: [PATCH 16/18] Handle CORS preflight and deploy policy selectors --- .github/scripts/deploy_cloud_run_candidate.sh | 3 + .../resolve_cloud_run_candidate_state.sh | 20 +- .../scripts/validate_cloud_run_deploy_env.sh | 21 +- .github/workflows/push.yml | 12 +- docs/engineering/skills/api-routes.md | 15 ++ docs/engineering/skills/testing.md | 13 +- docs/migration/stage-10-v2-policies.md | 20 ++ policyengine_api/asgi_factory.py | 38 ++-- tests/unit/test_asgi_factory.py | 198 ++++++++++++++++-- tests/unit/test_cloud_run_deploy_scripts.py | 97 ++++++++- 10 files changed, 377 insertions(+), 60 deletions(-) diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index 87418163b..76d49a190 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -20,6 +20,9 @@ env_vars=( "ROUTE_IMPL_HEALTH=${ROUTE_IMPL_HEALTH}" "ROUTE_IMPL_SPECIFICATION=${ROUTE_IMPL_SPECIFICATION}" "ROUTE_IMPL_METADATA=${ROUTE_IMPL_METADATA}" + "ROUTE_IMPL_POLICY=${ROUTE_IMPL_POLICY}" + "DB_READ_POLICY=${DB_READ_POLICY}" + "DB_WRITE_POLICY=${DB_WRITE_POLICY}" "SIM_COMPUTE_ECONOMY=old_gateway" "CLOUD_RUN_REVISION_TAG=${CLOUD_RUN_TAG}" "WEB_CONCURRENCY=${CLOUD_RUN_WEB_CONCURRENCY}" diff --git a/.github/scripts/resolve_cloud_run_candidate_state.sh b/.github/scripts/resolve_cloud_run_candidate_state.sh index d5f9af4c0..400ee62c3 100755 --- a/.github/scripts/resolve_cloud_run_candidate_state.sh +++ b/.github/scripts/resolve_cloud_run_candidate_state.sh @@ -78,26 +78,32 @@ image="$(jq -er ' | select(type == "string" and contains("@sha256:")) ' <<<"${revision_json}")" -route_selector_count=0 +deployment_selector_count=0 for selector in \ ROUTE_IMPL_HEALTH \ ROUTE_IMPL_SPECIFICATION \ - ROUTE_IMPL_METADATA; do + ROUTE_IMPL_METADATA \ + ROUTE_IMPL_POLICY \ + DB_READ_POLICY \ + DB_WRITE_POLICY; do if [[ -n "${!selector:-}" ]]; then - route_selector_count=$((route_selector_count + 1)) + deployment_selector_count=$((deployment_selector_count + 1)) fi done -if (( route_selector_count > 0 && route_selector_count < 3 )); then - echo "All Stage 6 route selectors are required when verifying candidate configuration" >&2 +if (( deployment_selector_count > 0 && deployment_selector_count < 6 )); then + echo "All route and policy database selectors are required when verifying candidate configuration" >&2 exit 2 fi -if (( route_selector_count == 3 )); then +if (( deployment_selector_count == 6 )); then for selector in \ ROUTE_IMPL_HEALTH \ ROUTE_IMPL_SPECIFICATION \ - ROUTE_IMPL_METADATA; do + ROUTE_IMPL_METADATA \ + ROUTE_IMPL_POLICY \ + DB_READ_POLICY \ + DB_WRITE_POLICY; do expected_value="${!selector}" actual_value="$(jq -r --arg name "${selector}" ' [ diff --git a/.github/scripts/validate_cloud_run_deploy_env.sh b/.github/scripts/validate_cloud_run_deploy_env.sh index 03a2f2403..33d4f92ec 100755 --- a/.github/scripts/validate_cloud_run_deploy_env.sh +++ b/.github/scripts/validate_cloud_run_deploy_env.sh @@ -41,6 +41,9 @@ cloud_run_require_env \ ROUTE_IMPL_HEALTH \ ROUTE_IMPL_SPECIFICATION \ ROUTE_IMPL_METADATA \ + ROUTE_IMPL_POLICY \ + DB_READ_POLICY \ + DB_WRITE_POLICY \ GATEWAY_AUTH_ISSUER \ GATEWAY_AUTH_AUDIENCE \ GATEWAY_AUTH_CLIENT_ID \ @@ -49,7 +52,8 @@ cloud_run_require_env \ for selector in \ ROUTE_IMPL_HEALTH \ ROUTE_IMPL_SPECIFICATION \ - ROUTE_IMPL_METADATA; do + ROUTE_IMPL_METADATA \ + ROUTE_IMPL_POLICY; do value="${!selector}" case "${value}" in flask_fallback|fastapi_native) ;; @@ -61,6 +65,21 @@ for selector in \ esac done +if [[ "${DB_READ_POLICY}" != "cloud_sql" ]]; then + printf '%s=%s is invalid; expected cloud_sql\n' \ + "DB_READ_POLICY" "${DB_READ_POLICY}" >&2 + exit 1 +fi + +case "${DB_WRITE_POLICY}" in + cloud_sql|dual_write) ;; + *) + printf '%s=%s is invalid; expected cloud_sql or dual_write\n' \ + "DB_WRITE_POLICY" "${DB_WRITE_POLICY}" >&2 + exit 1 + ;; +esac + selected_url_env="$( simulation_entrypoint_url_env_name "${SIM_ENTRYPOINT}" )" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 060780dcf..357c18e86 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -197,6 +197,9 @@ jobs: ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} + ROUTE_IMPL_POLICY: ${{ vars.ROUTE_IMPL_POLICY }} + DB_READ_POLICY: ${{ vars.DB_READ_POLICY }} + DB_WRITE_POLICY: ${{ vars.DB_WRITE_POLICY }} CLOUD_RUN_SERVICE: policyengine-api-staging CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: staging CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-staging-runtime-cache-url:latest @@ -235,7 +238,8 @@ jobs: run: make install - name: Run release tests run: >- - env -u ROUTE_IMPL_HEALTH -u ROUTE_IMPL_SPECIFICATION -u ROUTE_IMPL_METADATA + env -u ROUTE_IMPL_HEALTH -u ROUTE_IMPL_SPECIFICATION -u ROUTE_IMPL_METADATA -u ROUTE_IMPL_POLICY + -u DB_READ_POLICY -u DB_WRITE_POLICY -u CLOUD_RUN_SERVICE -u CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT -u CLOUD_RUN_RUNTIME_CACHE_URL_SECRET -u CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET -u CLOUD_RUN_MIN_INSTANCES -u CLOUD_RUN_SERVICE_MIN_INSTANCES -u CLOUD_RUN_MAX_INSTANCES -u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE make test env: @@ -331,6 +335,9 @@ jobs: ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} + ROUTE_IMPL_POLICY: ${{ vars.ROUTE_IMPL_POLICY }} + DB_READ_POLICY: ${{ vars.DB_READ_POLICY }} + DB_WRITE_POLICY: ${{ vars.DB_WRITE_POLICY }} permissions: contents: read id-token: write @@ -421,6 +428,9 @@ jobs: ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} + ROUTE_IMPL_POLICY: ${{ vars.ROUTE_IMPL_POLICY }} + DB_READ_POLICY: ${{ vars.DB_READ_POLICY }} + DB_WRITE_POLICY: ${{ vars.DB_WRITE_POLICY }} CLOUD_RUN_SERVICE: policyengine-api CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: production CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-prod-runtime-cache-url:latest diff --git a/docs/engineering/skills/api-routes.md b/docs/engineering/skills/api-routes.md index e0a293420..1b6a4b91a 100644 --- a/docs/engineering/skills/api-routes.md +++ b/docs/engineering/skills/api-routes.md @@ -86,6 +86,21 @@ The route's OpenAPI operation must expose every accepted query field with the same required status, type, default, and bounds enforced at runtime. Do not document query parameters accepted only by an untyped fallback parser. +## CORS Preflight Requests + +The outer ASGI application owns cross-origin request handling through +Starlette's `CORSMiddleware`. Do not add resource-specific `OPTIONS` operations +or CORS headers to FastAPI route functions. A browser preflight contains +`Origin` and `Access-Control-Request-Method`; the middleware must answer it +before resource routing, including when the eventual resource method is +`POST`, `PATCH`, or `DELETE`. An ordinary `OPTIONS` request without those +headers continues through normal route resolution. + +When a new public HTTP method or request header is added, verify that the +application-level CORS configuration permits it. Tests must cover the +preflight response and an error response, and browser-readable response +headers must be included in `Access-Control-Expose-Headers` when applicable. + ## Required Verification For each new or changed query contract, cover the applicable cases: diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 2bc9f2145..fa8b0d573 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -104,11 +104,14 @@ Redis server; missing managed-cache configuration fails closed; Cloud SQL and existing routes/compute remain primary; and generated migration-contract artifacts remain current. -Cloud Run must receive `ROUTE_IMPL_HEALTH`, `ROUTE_IMPL_SPECIFICATION`, and -`ROUTE_IMPL_METADATA` from the selected GitHub environment. Candidate -resolution must verify those values on the exact revision. Staging promotion -must wait for the complete Cloud Run staging integration suite against the -tagged candidate. +Cloud Run must receive `ROUTE_IMPL_HEALTH`, `ROUTE_IMPL_SPECIFICATION`, +`ROUTE_IMPL_METADATA`, `ROUTE_IMPL_POLICY`, `DB_READ_POLICY`, and +`DB_WRITE_POLICY` from the selected GitHub environment. Candidate resolution +must verify all six values on the exact revision. Staging promotion must wait +for the complete Cloud Run staging integration suite against the tagged +candidate. ASGI route tests must also prove that standard CORS middleware +answers browser preflight requests before route resolution and adds CORS +headers to error responses. For PR 3 Cloud Run candidate deployment changes, verify the command-building guards, workflow track structure, ASGI compatibility, and container build: diff --git a/docs/migration/stage-10-v2-policies.md b/docs/migration/stage-10-v2-policies.md index e9f4232f2..0e7b160db 100644 --- a/docs/migration/stage-10-v2-policies.md +++ b/docs/migration/stage-10-v2-policies.md @@ -53,6 +53,20 @@ confirm no metadata/schema difference. The relevant generated revisions are ## Activation +The GitHub `staging` and `production` environments must define all three Phase +10 deployment variables: + +```text +ROUTE_IMPL_POLICY=flask_fallback +DB_READ_POLICY=cloud_sql +DB_WRITE_POLICY=cloud_sql +``` + +The deployment workflow passes these values to Cloud Run, rejects missing or +invalid values before deployment, and verifies the exact values on the tagged +candidate revision before testing or promotion. These initial values do not +activate native policy readiness or v1 mirroring. + Native `/v2/policies` and `/v2/user-policies` routes use only the server-side Supabase connection. The routes are registered as preview resources; `ROUTE_IMPL_POLICY=fastapi_native` declares them operational for deployment @@ -106,6 +120,12 @@ operations internally. Native v2 routes and v1 mirroring return a secret-safe HTTP 503 for a timeout or other SQLAlchemy database failure so the caller can retry the complete request. +Browser preflight requests are handled by the outer ASGI CORS middleware before +FastAPI or Flask route resolution. It permits the public HTTP methods and +request headers used by v1 and v2, exposes `X-PolicyEngine-Request-Id`, and +applies CORS headers to typed errors and service-unavailable responses. Route +functions do not implement separate preflight behavior. + ## Monitoring Monitor structured events with metric names `v1_policy_mirror_operations` and diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index c11eb1a78..bcc9802ed 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -35,29 +35,17 @@ generate_request_id, ) from starlette.datastructures import MutableHeaders +from starlette.middleware.cors import CORSMiddleware from starlette.middleware.gzip import GZipMiddleware from starlette.responses import PlainTextResponse, Response +from starlette.types import ASGIApp -def _add_vary_origin(response) -> None: - vary = response.headers.get("Vary") - if vary is None: - response.headers["Vary"] = "Origin" - return - if "origin" not in {value.strip().lower() for value in vary.split(",")}: - response.headers["Vary"] = f"{vary}, Origin" - - -def _apply_shared_response_headers( - request: Request, +def _apply_request_id_header( response: Response, request_id: str, ) -> None: response.headers[REQUEST_ID_HEADER] = request_id - origin = request.headers.get("origin") - if origin and "access-control-allow-origin" not in response.headers: - response.headers["Access-Control-Allow-Origin"] = origin - _add_vary_origin(response) def create_asgi_app( @@ -66,7 +54,7 @@ def create_asgi_app( route_settings: RouteImplementationSettings | None = None, dependencies: NativeRouteDependencies | None = None, shutdown_callback: Callable[[], None] | None = None, -) -> FastAPI: +) -> ASGIApp: """Create the Stage 2 FastAPI shell around the existing Flask app.""" if route_settings is None: @@ -108,7 +96,7 @@ async def add_headers_to_unhandled_errors( "policyengine_request_id", request.headers.get(REQUEST_ID_HEADER) or generate_request_id(), ) - _apply_shared_response_headers(request, response, request_id) + _apply_request_id_header(response, request_id) return response @app.exception_handler(RequestValidationError) @@ -128,7 +116,7 @@ async def oversized_v2_request( return v2_error_response(413, str(error)) @app.middleware("http") - async def add_cors_for_native_routes(request, call_next): + async def add_request_context_and_migration_logging(request, call_next): started_at = time.time() request_id = request.headers.get(REQUEST_ID_HEADER) or generate_request_id() MutableHeaders(scope=request.scope)[REQUEST_ID_HEADER] = request_id @@ -160,7 +148,7 @@ def log_native_route(status_code: int) -> None: except Exception: log_native_route(500) raise - _apply_shared_response_headers(request, response, request_id) + _apply_request_id_header(response, request_id) log_native_route(response.status_code) return response finally: @@ -176,4 +164,14 @@ def log_native_route(status_code: int) -> None: app.include_router(build_metadata_router(dependencies)) app.mount("/", WSGIMiddleware(wsgi_app)) - return app + # The public API already permits every web origin. Use the standard ASGI + # implementation while preserving the existing reflected-origin response. + return CORSMiddleware( + app=app, + allow_origin_regex=".*", + allow_methods=["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"], + allow_headers=["*"], + expose_headers=[REQUEST_ID_HEADER], + allow_credentials=False, + max_age=600, + ) diff --git a/tests/unit/test_asgi_factory.py b/tests/unit/test_asgi_factory.py index 92d2649ce..997eb8e04 100644 --- a/tests/unit/test_asgi_factory.py +++ b/tests/unit/test_asgi_factory.py @@ -3,6 +3,7 @@ import sys import threading from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace from types import SimpleNamespace from unittest.mock import Mock, patch @@ -11,7 +12,7 @@ from fastapi.testclient import TestClient from flask import Flask, Response, jsonify, make_response, request from flask_cors import CORS -from policyengine_api.asgi_factory import _add_vary_origin, create_asgi_app +from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, @@ -20,7 +21,6 @@ REQUEST_ID_HEADER, current_request_id, ) -from starlette.responses import Response as ASGIResponse def create_test_wsgi_app() -> Flask: @@ -367,25 +367,6 @@ def make_request(request_id): } -@pytest.mark.parametrize( - ("existing_vary", "expected_vary"), - [ - (None, "Origin"), - ("Accept-Encoding", "Accept-Encoding, Origin"), - ("Origin", "Origin"), - ("Accept-Encoding, origin", "Accept-Encoding, origin"), - ], -) -def test_add_vary_origin_preserves_existing_values(existing_vary, expected_vary): - response = ASGIResponse() - if existing_vary is not None: - response.headers["Vary"] = existing_vary - - _add_vary_origin(response) - - assert response.headers["Vary"] == expected_vary - - def test_asgi_entrypoint_imports_and_serves_health(monkeypatch): monkeypatch.setenv("FLASK_DEBUG", "1") sys.modules.pop("policyengine_api.asgi", None) @@ -498,6 +479,181 @@ def test_health_route_uses_same_reflected_cors_policy(): assert response.headers["vary"] == "Origin" +@pytest.mark.parametrize( + ("path", "method"), + [ + ("/v2/policies?country_id=us", "POST"), + ( + "/v2/user-policies/00000000-0000-0000-0000-000000000001?country_id=us", + "PATCH", + ), + ( + "/v2/user-policies/00000000-0000-0000-0000-000000000001?country_id=us", + "DELETE", + ), + ], +) +def test_cors_preflight_is_handled_before_v2_route_resolution(path, method): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.options( + path, + headers={ + "Origin": "https://app.policyengine.org", + "Access-Control-Request-Method": method, + "Access-Control-Request-Headers": ( + "authorization, content-type, x-policyengine-request-id" + ), + }, + ) + + assert response.status_code == 200 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + assert method in response.headers["access-control-allow-methods"].split(", ") + assert response.headers["access-control-allow-headers"] == ( + "authorization, content-type, x-policyengine-request-id" + ) + assert response.headers["access-control-max-age"] == "600" + assert response.headers["vary"] == "Origin" + + +def test_cors_preflight_rejects_a_method_outside_the_public_http_contract(): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.options( + "/v2/policies?country_id=us", + headers={ + "Origin": "https://app.policyengine.org", + "Access-Control-Request-Method": "TRACE", + }, + ) + + assert response.status_code == 400 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + + +def test_cors_preflight_does_not_invoke_the_mounted_flask_application(): + def failing_wsgi_app(_environ, _start_response): + raise AssertionError("preflight reached Flask") + + client = TestClient(create_asgi_app(failing_wsgi_app)) + + response = client.options( + "/fallback", + headers={ + "Origin": "https://app.policyengine.org", + "Access-Control-Request-Method": "GET", + }, + ) + + assert response.status_code == 200 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + + +def test_non_cors_options_request_uses_normal_v2_method_routing(): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.options("/v2/policies?country_id=us") + + assert response.status_code == 405 + assert response.json() == { + "status": "error", + "message": "API v2 resource 'policies' does not support this method", + } + + +@pytest.mark.parametrize( + ("method", "path", "expected_status"), + [ + ("GET", "/v2/policies", 422), + ("GET", "/v2/not-a-resource", 404), + ("OPTIONS", "/v2/policies?country_id=us", 405), + ], +) +def test_cors_headers_are_present_on_v2_error_responses( + method, + path, + expected_status, +): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.request( + method, + path, + headers={"Origin": "https://app.policyengine.org"}, + ) + + assert response.status_code == expected_status + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + assert REQUEST_ID_HEADER.lower() in { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + + +def test_cors_exposes_response_request_id_to_browser_clients(): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.get( + "/health", + headers={ + "Origin": "https://app.policyengine.org", + REQUEST_ID_HEADER: "browser-request-id", + }, + ) + + assert response.headers[REQUEST_ID_HEADER] == "browser-request-id" + exposed_headers = { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + assert REQUEST_ID_HEADER.lower() in exposed_headers + + +def test_cors_headers_are_present_on_unhandled_native_errors(): + def raise_unhandled_error() -> bool: + raise RuntimeError("unavailable") + + dependencies = replace( + _stage6_dependencies(), + readiness_probe=raise_unhandled_error, + ) + client = TestClient( + create_asgi_app( + create_test_wsgi_app(), + route_settings=_stage6_settings(health=RouteImplementation.FASTAPI_NATIVE), + dependencies=dependencies, + ), + raise_server_exceptions=False, + ) + response = client.get( + "/readiness-check", + headers={ + "Origin": "https://app.policyengine.org", + REQUEST_ID_HEADER: "failed-request-id", + }, + ) + + assert response.status_code == 500 + assert response.headers[REQUEST_ID_HEADER] == "failed-request-id" + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + exposed_headers = { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + assert REQUEST_ID_HEADER.lower() in exposed_headers + + def test_public_simulation_gateway_health_probe_checks_gateway(): client = TestClient(create_asgi_app(create_test_wsgi_app())) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index d6486030a..2efd748ca 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -85,6 +85,9 @@ def _required_runtime_env() -> dict[str, str]: "ROUTE_IMPL_HEALTH": "fastapi_native", "ROUTE_IMPL_SPECIFICATION": "fastapi_native", "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", **_v2_target_env(), **_gateway_auth_env(), } @@ -116,6 +119,9 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: "ROUTE_IMPL_HEALTH": "fastapi_native", "ROUTE_IMPL_SPECIFICATION": "fastapi_native", "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", }, "updates": [], } @@ -526,6 +532,9 @@ def test_validate_cloud_run_deploy_env_accepts_direct_mode_from_environment(): ROUTE_IMPL_HEALTH="fastapi_native", ROUTE_IMPL_SPECIFICATION="fastapi_native", ROUTE_IMPL_METADATA="fastapi_native", + ROUTE_IMPL_POLICY="flask_fallback", + DB_READ_POLICY="cloud_sql", + DB_WRITE_POLICY="cloud_sql", POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, **_v2_target_env(), **_gateway_auth_env(), @@ -541,9 +550,12 @@ def test_validate_cloud_run_deploy_env_accepts_direct_mode_from_environment(): "ROUTE_IMPL_HEALTH", "ROUTE_IMPL_SPECIFICATION", "ROUTE_IMPL_METADATA", + "ROUTE_IMPL_POLICY", + "DB_READ_POLICY", + "DB_WRITE_POLICY", ], ) -def test_validate_cloud_run_deploy_env_requires_stage6_selectors(missing_selector): +def test_validate_cloud_run_deploy_env_requires_migration_selectors(missing_selector): env = _script_env(**_required_runtime_env()) env.pop(missing_selector) @@ -562,9 +574,10 @@ def test_validate_cloud_run_deploy_env_requires_stage6_selectors(missing_selecto "ROUTE_IMPL_HEALTH", "ROUTE_IMPL_SPECIFICATION", "ROUTE_IMPL_METADATA", + "ROUTE_IMPL_POLICY", ], ) -def test_validate_cloud_run_deploy_env_rejects_invalid_stage6_selectors( +def test_validate_cloud_run_deploy_env_rejects_invalid_route_selectors( invalid_selector, ): env = _script_env(**_required_runtime_env()) @@ -582,6 +595,33 @@ def test_validate_cloud_run_deploy_env_rejects_invalid_stage6_selectors( ) in result.stderr +@pytest.mark.parametrize( + ("selector", "invalid_value", "expected_values"), + [ + ("DB_READ_POLICY", "supabase", "cloud_sql"), + ("DB_WRITE_POLICY", "supabase", "cloud_sql or dual_write"), + ], +) +def test_validate_cloud_run_deploy_env_rejects_invalid_policy_database_selectors( + selector, + invalid_value, + expected_values, +): + env = _script_env(**_required_runtime_env()) + env[selector] = invalid_value + + result = _run_script( + ".github/scripts/validate_cloud_run_deploy_env.sh", + env, + ) + + assert result.returncode == 1 + assert ( + f"{selector}={invalid_value} is invalid; expected {expected_values}" + in result.stderr + ) + + @pytest.mark.parametrize( ("entrypoint", "selected_url_env", "selected_url"), [ @@ -607,6 +647,9 @@ def test_validate_cloud_run_deploy_env_requires_only_selected_url( ROUTE_IMPL_HEALTH="fastapi_native", ROUTE_IMPL_SPECIFICATION="fastapi_native", ROUTE_IMPL_METADATA="fastapi_native", + ROUTE_IMPL_POLICY="flask_fallback", + DB_READ_POLICY="cloud_sql", + DB_WRITE_POLICY="cloud_sql", POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, **_v2_target_env(), **_gateway_auth_env(), @@ -776,6 +819,9 @@ def test_deploy_cloud_run_candidate_dry_run_never_shifts_traffic(): "ROUTE_IMPL_METADATA", ): assert result.stdout.count(f"{selector}=fastapi_native") == 1 + assert result.stdout.count("ROUTE_IMPL_POLICY=flask_fallback") == 1 + assert result.stdout.count("DB_READ_POLICY=cloud_sql") == 1 + assert result.stdout.count("DB_WRITE_POLICY=cloud_sql") == 1 def test_staging_and_production_use_distinct_cloud_run_runtime_identities(): @@ -952,13 +998,16 @@ def test_resolve_cloud_run_candidate_rejects_changed_image(tmp_path): assert "Candidate image changed" in result.stderr -def test_resolve_cloud_run_candidate_verifies_stage6_route_selectors(tmp_path): +def test_resolve_cloud_run_candidate_verifies_deployment_selectors(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) env = { **_fake_gcloud_env(gcloud_path, state_path), "ROUTE_IMPL_HEALTH": "fastapi_native", "ROUTE_IMPL_SPECIFICATION": "fastapi_native", "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", } result = _run_script( @@ -969,7 +1018,7 @@ def test_resolve_cloud_run_candidate_verifies_stage6_route_selectors(tmp_path): assert result.returncode == 0, result.stderr -def test_resolve_cloud_run_candidate_rejects_stage6_selector_mismatch(tmp_path): +def test_resolve_cloud_run_candidate_rejects_deployment_selector_mismatch(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) state = json.loads(state_path.read_text(encoding="utf-8")) state["candidate_env"]["ROUTE_IMPL_METADATA"] = "flask_fallback" @@ -982,6 +1031,9 @@ def test_resolve_cloud_run_candidate_rejects_stage6_selector_mismatch(tmp_path): "ROUTE_IMPL_HEALTH": "fastapi_native", "ROUTE_IMPL_SPECIFICATION": "fastapi_native", "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", }, ) @@ -992,6 +1044,34 @@ def test_resolve_cloud_run_candidate_rejects_stage6_selector_mismatch(tmp_path): ) in result.stderr +def test_resolve_cloud_run_candidate_rejects_policy_write_selector_mismatch( + tmp_path, +): + gcloud_path, state_path = _fake_gcloud(tmp_path) + state = json.loads(state_path.read_text(encoding="utf-8")) + state["candidate_env"]["DB_WRITE_POLICY"] = "dual_write" + state_path.write_text(json.dumps(state), encoding="utf-8") + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + { + **_fake_gcloud_env(gcloud_path, state_path), + "ROUTE_IMPL_HEALTH": "fastapi_native", + "ROUTE_IMPL_SPECIFICATION": "fastapi_native", + "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", + }, + ) + + assert result.returncode == 2 + assert ( + "Revision policyengine-api-00002-new has DB_WRITE_POLICY=dual_write; " + "expected cloud_sql" + ) in result.stderr + + def test_set_cloud_run_revision_promotes_and_rolls_back_exact_revisions(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) env = _fake_gcloud_env(gcloud_path, state_path) @@ -1389,6 +1469,9 @@ def test_push_workflow_uses_local_redis_for_predeployment_test_suite(): assert "RUNTIME_CACHE_ENVIRONMENT: test" in test_step assert "RUNTIME_CACHE_SERVICE: api" in test_step assert "-u ROUTE_IMPL_HEALTH" in test_step + assert "-u ROUTE_IMPL_POLICY" in test_step + assert "-u DB_READ_POLICY" in test_step + assert "-u DB_WRITE_POLICY" in test_step assert "-u CLOUD_RUN_SERVICE" in test_step assert "-u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE" in test_step @@ -1465,16 +1548,20 @@ def test_workflows_scope_simulation_routing_config_to_github_environments(): assert secret_env in job -def test_cloud_run_deploy_jobs_use_environment_scoped_stage6_route_selectors(): +def test_cloud_run_candidate_jobs_use_environment_scoped_migration_selectors(): workflow = _push_workflow() selectors = ( "ROUTE_IMPL_HEALTH", "ROUTE_IMPL_SPECIFICATION", "ROUTE_IMPL_METADATA", + "ROUTE_IMPL_POLICY", + "DB_READ_POLICY", + "DB_WRITE_POLICY", ) for job_name, environment in ( ("deploy-cloud-run-staging", "staging"), + ("promote-cloud-run-staging", "staging"), ("deploy-cloud-run-candidate", "production"), ): job = _workflow_job_block(workflow, job_name) From f410bcac1bf50501409730ab61b880826716de4f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:43:42 +0400 Subject: [PATCH 17/18] Isolate staging databases and verify Phase 10 rollback --- .github/scripts/cloud_run_env.sh | 2 +- .github/scripts/deploy_cloud_run_candidate.sh | 2 +- .github/scripts/migrate_v1_cloud_sql.sh | 8 +- .../resolve_cloud_run_candidate_state.sh | 51 ++ .github/scripts/run_phase10_staging_probe.sh | 34 ++ .../scripts/validate_cloud_run_deploy_env.sh | 2 + .../scripts/validate_database_environment.sh | 162 ++++++ .../validate_phase10_staging_exercise_env.sh | 49 ++ .github/workflows/migrate-v1-cloud-sql.yml | 67 +++ .github/workflows/push.yml | 296 +++++++++-- .github/workflows/seed-v2-database.yml | 8 + docs/migration/stage-10-v2-policies.md | 39 ++ .../data/v2/policy_migration_qualification.py | 100 +++- scripts/qualify_v2_policy_migration.py | 2 +- .../integration/test_live_phase10_staging.py | 498 ++++++++++++++++++ tests/integration/test_live_v2_policies.py | 188 +++++++ tests/unit/test_alembic_workflows.py | 69 ++- tests/unit/test_app_engine_decommission.py | 10 +- tests/unit/test_cloud_run_deploy_scripts.py | 260 ++++++++- tests/unit/v2/test_metadata_deployment.py | 16 +- .../v2/test_policy_migration_qualification.py | 45 +- 21 files changed, 1817 insertions(+), 91 deletions(-) create mode 100644 .github/scripts/run_phase10_staging_probe.sh create mode 100644 .github/scripts/validate_database_environment.sh create mode 100644 .github/scripts/validate_phase10_staging_exercise_env.sh create mode 100644 .github/workflows/migrate-v1-cloud-sql.yml create mode 100644 tests/integration/test_live_phase10_staging.py create mode 100644 tests/integration/test_live_v2_policies.py diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index 37522c53d..c97672628 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -36,7 +36,7 @@ cloud_run_set_defaults() { # but adds no real scale-out delay because boot exceeds it either way. Sizing # rationale in docs/migration/cloud-run-operations.md. CLOUD_RUN_STARTUP_PROBE="${CLOUD_RUN_STARTUP_PROBE:-httpGet.path=/readiness-check,httpGet.port=${CLOUD_RUN_PORT},initialDelaySeconds=240,periodSeconds=10,failureThreshold=24,timeoutSeconds=5}" - CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}" + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}" CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}" diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index 76d49a190..8136894e7 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -66,7 +66,7 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \ --subnet "${CLOUD_RUN_VPC_SUBNET}" \ --vpc-egress "${CLOUD_RUN_VPC_EGRESS}" \ --service-account "${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \ - --add-cloudsql-instances "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" \ + --set-cloudsql-instances "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" \ --port "${CLOUD_RUN_PORT}" \ --cpu "${CLOUD_RUN_CPU}" \ --cpu-boost \ diff --git a/.github/scripts/migrate_v1_cloud_sql.sh b/.github/scripts/migrate_v1_cloud_sql.sh index 77d2bb23d..ae4928d0b 100644 --- a/.github/scripts/migrate_v1_cloud_sql.sh +++ b/.github/scripts/migrate_v1_cloud_sql.sh @@ -3,15 +3,19 @@ set -euo pipefail : "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME:?POLICYENGINE_DB_INSTANCE_CONNECTION_NAME is required}" +: "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET:?POLICYENGINE_DB_READONLY_PASSWORD_SECRET is required}" +: "${POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET:?POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET is required}" + +bash .github/scripts/validate_database_environment.sh cloud-sql readonly_password="$( gcloud secrets versions access latest \ - --secret policyengine-api-prod-db-readonly-password \ + --secret "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" \ --project policyengine-api )" migration_password="$( gcloud secrets versions access latest \ - --secret policyengine-api-prod-db-migration-password \ + --secret "${POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" \ --project policyengine-api )" diff --git a/.github/scripts/resolve_cloud_run_candidate_state.sh b/.github/scripts/resolve_cloud_run_candidate_state.sh index 400ee62c3..b6e200084 100755 --- a/.github/scripts/resolve_cloud_run_candidate_state.sh +++ b/.github/scripts/resolve_cloud_run_candidate_state.sh @@ -122,6 +122,57 @@ if (( deployment_selector_count == 6 )); then done fi +database_identity_count=0 +for setting in \ + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME \ + V2_SUPABASE_PROJECT_REF \ + V2_SUPABASE_ENVIRONMENT \ + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE; do + if [[ -n "${!setting:-}" ]]; then + database_identity_count=$((database_identity_count + 1)) + fi +done + +if (( database_identity_count > 0 && database_identity_count < 4 )); then + echo "All database identity settings are required when verifying candidate configuration" >&2 + exit 2 +fi + +if (( database_identity_count == 4 )); then + for setting in \ + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME \ + V2_SUPABASE_PROJECT_REF \ + V2_SUPABASE_ENVIRONMENT \ + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE; do + expected_value="${!setting}" + actual_value="$(jq -r --arg name "${setting}" ' + [ + .spec.containers[0].env[]? + | select(.name == $name) + | .value + ] + | if length == 1 then .[0] else "" end + ' <<<"${revision_json}")" + if [[ "${actual_value}" != "${expected_value}" ]]; then + printf 'Revision %s has %s=%s; expected %s\n' \ + "${revision}" "${setting}" "${actual_value:-}" \ + "${expected_value}" >&2 + exit 2 + fi + done + + attached_cloud_sql="$(jq -r ' + .metadata.annotations["run.googleapis.com/cloudsql-instances"] // empty + ' <<<"${revision_json}")" + if [[ "${attached_cloud_sql}" != \ + "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" ]]; then + printf 'Revision %s attaches Cloud SQL instance %s; expected %s\n' \ + "${revision}" "${attached_cloud_sql:-}" \ + "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" >&2 + exit 2 + fi +fi + if [[ -n "${CLOUD_RUN_EXPECTED_REVISION:-}" \ && "${revision}" != "${CLOUD_RUN_EXPECTED_REVISION}" ]]; then printf 'Candidate tag %s moved: expected revision %s, found %s\n' \ diff --git a/.github/scripts/run_phase10_staging_probe.sh b/.github/scripts/run_phase10_staging_probe.sh new file mode 100644 index 000000000..00ab3b6ee --- /dev/null +++ b/.github/scripts/run_phase10_staging_probe.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET:?POLICYENGINE_DB_READONLY_PASSWORD_SECRET is required}" + +case "${1:-}" in + activation) + test_name="test_live_phase10_activation_failure_and_retry" + ;; + rollback) + test_name="test_live_phase10_cloud_sql_only_rollback" + ;; + *) + echo "Usage: $0 {activation|rollback}" >&2 + exit 1 + ;; +esac + +readonly_password="$( + gcloud secrets versions access latest \ + --secret "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" \ + --project policyengine-api +)" +if [[ -z "${readonly_password}" ]]; then + echo "The staging Cloud SQL read-only password must not be empty." >&2 + exit 1 +fi + +printf '::add-mask::%s\n' "${readonly_password}" +POLICYENGINE_DB_READONLY_PASSWORD="${readonly_password}" \ + python -m pytest \ + "tests/integration/test_live_phase10_staging.py::${test_name}" \ + -v diff --git a/.github/scripts/validate_cloud_run_deploy_env.sh b/.github/scripts/validate_cloud_run_deploy_env.sh index 33d4f92ec..1ccd1e665 100755 --- a/.github/scripts/validate_cloud_run_deploy_env.sh +++ b/.github/scripts/validate_cloud_run_deploy_env.sh @@ -6,6 +6,8 @@ source .github/scripts/cloud_run_env.sh source .github/scripts/simulation_entrypoint_env.sh cloud_run_set_defaults +bash .github/scripts/validate_database_environment.sh runtime + # Cloud Run rejects deploys where the traffic tag and service name together # exceed 46 characters (they form the tag URL's DNS label). Fail fast here # with a clear message instead of at gcloud. diff --git a/.github/scripts/validate_database_environment.sh b/.github/scripts/validate_database_environment.sh new file mode 100644 index 000000000..c619cd84b --- /dev/null +++ b/.github/scripts/validate_database_environment.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +set -euo pipefail + +mode="${1:-}" + +require_environment_variables() { + local missing=() + local name + + for name in "$@"; do + if [[ -z "${!name:-}" ]]; then + missing+=("${name}") + fi + done + + if (( ${#missing[@]} > 0 )); then + echo "Missing required database environment configuration: ${missing[*]}" >&2 + return 1 + fi +} + +require_environment_variables DEPLOYMENT_ENVIRONMENT + +case "${DEPLOYMENT_ENVIRONMENT}" in + staging|production) ;; + *) + printf 'DEPLOYMENT_ENVIRONMENT=%s is invalid; expected staging or production\n' \ + "${DEPLOYMENT_ENVIRONMENT}" >&2 + exit 1 + ;; +esac + +validate_cloud_sql() { + require_environment_variables \ + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME \ + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME + + if [[ "${DEPLOYMENT_ENVIRONMENT}" == "staging" ]]; then + if [[ "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" == \ + "${PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" ]]; then + echo "Staging Cloud SQL must use an instance distinct from production." >&2 + return 1 + fi + elif [[ "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" != \ + "${PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" ]]; then + echo "Production Cloud SQL does not match the configured production instance." >&2 + return 1 + fi + + local credential_setting_count=0 + local setting + for setting in \ + POLICYENGINE_DB_READONLY_PASSWORD_SECRET \ + POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET \ + PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET \ + PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET; do + if [[ -n "${!setting:-}" ]]; then + credential_setting_count=$((credential_setting_count + 1)) + fi + done + if (( credential_setting_count > 0 && credential_setting_count < 4 )); then + echo "All Cloud SQL migration credential resources are required together." >&2 + return 1 + fi + if (( credential_setting_count == 4 )); then + if [[ "${DEPLOYMENT_ENVIRONMENT}" == "staging" ]]; then + if [[ "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" == \ + "${PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" || \ + "${POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" == \ + "${PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" ]]; then + echo "Staging Cloud SQL migration credentials must be distinct from production." >&2 + return 1 + fi + elif [[ "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" != \ + "${PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" || \ + "${POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" != \ + "${PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" ]]; then + echo "Production Cloud SQL migration credentials do not match production resources." >&2 + return 1 + fi + fi +} + +validate_supabase() { + require_environment_variables \ + V2_SUPABASE_PROJECT_REF \ + V2_SUPABASE_ENVIRONMENT \ + PRODUCTION_V2_SUPABASE_PROJECT_REF + + if [[ "${DEPLOYMENT_ENVIRONMENT}" == "staging" ]]; then + if [[ "${V2_SUPABASE_ENVIRONMENT}" != "staging" ]]; then + echo "Staging V2_SUPABASE_ENVIRONMENT must be exactly staging." >&2 + return 1 + fi + if [[ "${V2_SUPABASE_PROJECT_REF}" == \ + "${PRODUCTION_V2_SUPABASE_PROJECT_REF}" ]]; then + echo "Staging Supabase must use a project distinct from production." >&2 + return 1 + fi + else + if [[ "${V2_SUPABASE_PROJECT_REF}" != \ + "${PRODUCTION_V2_SUPABASE_PROJECT_REF}" ]]; then + echo "Production Supabase does not match the configured production project." >&2 + return 1 + fi + if [[ "${V2_SUPABASE_ENVIRONMENT}" == "staging" ]]; then + echo "Production V2_SUPABASE_ENVIRONMENT must not be staging." >&2 + return 1 + fi + fi +} + +validate_runtime_secret() { + require_environment_variables \ + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \ + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \ + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE \ + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE + + if [[ "${DEPLOYMENT_ENVIRONMENT}" == "staging" ]]; then + if [[ "${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" == \ + "${PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" ]]; then + echo "Staging v1 runtime must use a database secret distinct from production." >&2 + return 1 + fi + if [[ "${V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" == \ + "${PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" ]]; then + echo "Staging v2 runtime must use a database secret distinct from production." >&2 + return 1 + fi + else + if [[ "${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" != \ + "${PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" ]]; then + echo "Production v1 runtime does not match the configured production secret." >&2 + return 1 + fi + if [[ "${V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" != \ + "${PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" ]]; then + echo "Production v2 runtime does not match the configured production secret." >&2 + return 1 + fi + fi +} + +case "${mode}" in + cloud-sql) + validate_cloud_sql + ;; + supabase) + validate_supabase + ;; + runtime) + validate_cloud_sql + validate_supabase + validate_runtime_secret + ;; + *) + echo "Usage: $0 {cloud-sql|supabase|runtime}" >&2 + exit 1 + ;; +esac diff --git a/.github/scripts/validate_phase10_staging_exercise_env.sh b/.github/scripts/validate_phase10_staging_exercise_env.sh new file mode 100644 index 000000000..f63522c95 --- /dev/null +++ b/.github/scripts/validate_phase10_staging_exercise_env.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euo pipefail + +required=( + DEPLOYMENT_ENVIRONMENT + ROUTE_IMPL_POLICY + DB_READ_POLICY + DB_WRITE_POLICY + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE + V2_FAILURE_DATABASE_URL_SECRET_RESOURCE + POLICYENGINE_DB_READONLY_PASSWORD_SECRET +) + +for setting in "${required[@]}"; do + if [[ -z "${!setting:-}" ]]; then + printf '%s is required for the Phase 10 staging exercise\n' "${setting}" >&2 + exit 1 + fi +done + +if [[ "${DEPLOYMENT_ENVIRONMENT}" != "staging" ]]; then + echo "The Phase 10 live exercise may run only against staging." >&2 + exit 1 +fi +if [[ "${ROUTE_IMPL_POLICY}" != "fastapi_native" ]]; then + echo "ROUTE_IMPL_POLICY must be fastapi_native for the Phase 10 exercise." >&2 + exit 1 +fi +if [[ "${DB_READ_POLICY}" != "cloud_sql" ]]; then + echo "DB_READ_POLICY must remain cloud_sql for the Phase 10 exercise." >&2 + exit 1 +fi +if [[ "${DB_WRITE_POLICY}" != "cloud_sql" ]]; then + echo "The staging candidate must begin with DB_WRITE_POLICY=cloud_sql." >&2 + exit 1 +fi +if [[ "${V2_FAILURE_DATABASE_URL_SECRET_RESOURCE}" == \ + "${V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" ]]; then + echo "The controlled-failure database secret must differ from the valid staging secret." >&2 + exit 1 +fi +if [[ "${V2_FAILURE_DATABASE_URL_SECRET_RESOURCE}" == \ + "${PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE:-}" ]]; then + echo "The controlled-failure database secret must not identify production." >&2 + exit 1 +fi + +bash .github/scripts/validate_database_environment.sh runtime diff --git a/.github/workflows/migrate-v1-cloud-sql.yml b/.github/workflows/migrate-v1-cloud-sql.yml new file mode 100644 index 000000000..fbf22fe57 --- /dev/null +++ b/.github/workflows/migrate-v1-cloud-sql.yml @@ -0,0 +1,67 @@ +name: Migrate v1 Cloud SQL + +on: + workflow_call: + inputs: + deployment_environment: + description: Logical database environment + required: true + type: string + github_environment: + description: GitHub environment containing the Cloud SQL target + required: true + type: string + workflow_dispatch: + inputs: + deployment_environment: + description: Logical database environment + required: true + type: choice + options: + - staging + - production + github_environment: + description: GitHub environment containing the Cloud SQL target + required: true + type: environment + +jobs: + migrate: + name: Upgrade and verify v1 Cloud SQL + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: ${{ inputs.github_environment }} + permissions: + contents: read + id-token: write + env: + DEPLOYMENT_ENVIRONMENT: ${{ inputs.deployment_environment }} + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + POLICYENGINE_DB_READONLY_PASSWORD_SECRET: ${{ vars.POLICYENGINE_DB_READONLY_PASSWORD_SECRET }} + POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET: ${{ vars.POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET }} + PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET: ${{ vars.PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET }} + PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET: ${{ vars.PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Authenticate to GCP + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }} + - name: Set up GCloud + uses: google-github-actions/setup-gcloud@v2 + - name: Install dependencies + run: make install + - name: Start Cloud SQL Auth Proxy + run: bash .github/scripts/start_cloud_sql_proxy.sh + - name: Upgrade and verify v1 database + run: bash .github/scripts/migrate_v1_cloud_sql.sh + - name: Stop Cloud SQL Auth Proxy + if: always() + run: bash .github/scripts/stop_cloud_sql_proxy.sh diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 357c18e86..816af143a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -126,50 +126,25 @@ jobs: - name: Publish Git Tag run: ".github/publish-git-tag.sh" - migrate-v1-cloud-sql: - name: Upgrade v1 Cloud SQL schema - runs-on: ubuntu-latest + migrate-v1-staging-cloud-sql: + name: Upgrade staging v1 Cloud SQL schema needs: - ensure-staging-model-version-aligns-with-sim-api - publish-git-tag if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') - environment: production-database - permissions: - contents: read - id-token: write - env: - POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Authenticate to GCP - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }} - - name: Set up GCloud - uses: google-github-actions/setup-gcloud@v2 - - name: Install dependencies - run: make install - - name: Start Cloud SQL Auth Proxy - run: bash .github/scripts/start_cloud_sql_proxy.sh - - name: Upgrade and verify v1 database - run: bash .github/scripts/migrate_v1_cloud_sql.sh - - name: Stop Cloud SQL Auth Proxy - if: always() - run: bash .github/scripts/stop_cloud_sql_proxy.sh + uses: ./.github/workflows/migrate-v1-cloud-sql.yml + with: + deployment_environment: staging + github_environment: staging-database + secrets: inherit seed-v2-staging-database: name: Seed staging v2 database needs: - publish-git-tag - - migrate-v1-cloud-sql + - migrate-v1-staging-cloud-sql if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -184,16 +159,18 @@ jobs: needs: - ensure-staging-model-version-aligns-with-sim-api - publish-git-tag - - migrate-v1-cloud-sql + - migrate-v1-staging-cloud-sql - seed-v2-staging-database if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') environment: staging env: + DEPLOYMENT_ENVIRONMENT: staging SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} @@ -204,9 +181,13 @@ jobs: CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: staging CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-staging-runtime-cache-url:latest CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-staging-runtime-cache-ca:latest + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + PRODUCTION_V2_SUPABASE_PROJECT_REF: ${{ vars.PRODUCTION_V2_SUPABASE_PROJECT_REF }} V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} # Staging stays scale-to-zero, single instance: it exists for per-push # validation, not capacity. Both the revision-level (--min-instances) and # service-level (--min) floors are 0. @@ -238,10 +219,9 @@ jobs: run: make install - name: Run release tests run: >- - env -u ROUTE_IMPL_HEALTH -u ROUTE_IMPL_SPECIFICATION -u ROUTE_IMPL_METADATA -u ROUTE_IMPL_POLICY - -u DB_READ_POLICY -u DB_WRITE_POLICY - -u CLOUD_RUN_SERVICE -u CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT -u CLOUD_RUN_RUNTIME_CACHE_URL_SECRET -u CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET -u CLOUD_RUN_MIN_INSTANCES -u CLOUD_RUN_SERVICE_MIN_INSTANCES -u CLOUD_RUN_MAX_INSTANCES - -u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE make test + env -u DEPLOYMENT_ENVIRONMENT -u ROUTE_IMPL_HEALTH -u ROUTE_IMPL_SPECIFICATION -u ROUTE_IMPL_METADATA -u ROUTE_IMPL_POLICY -u DB_READ_POLICY -u DB_WRITE_POLICY + -u CLOUD_RUN_SERVICE -u CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT -u CLOUD_RUN_RUNTIME_CACHE_URL_SECRET -u CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET -u CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET -u CLOUD_RUN_MIN_INSTANCES -u CLOUD_RUN_SERVICE_MIN_INSTANCES -u CLOUD_RUN_MAX_INSTANCES + -u V2_SUPABASE_PROJECT_REF -u V2_SUPABASE_ENVIRONMENT -u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE -u PRODUCTION_V2_SUPABASE_PROJECT_REF -u PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE -u PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME make test env: RUNTIME_CACHE_MODE: local RUNTIME_CACHE_URL: redis://127.0.0.1:6379/0 @@ -305,6 +285,7 @@ jobs: if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') + environment: staging steps: - name: Checkout repo uses: actions/checkout@v4 @@ -313,12 +294,13 @@ jobs: with: python-version: "3.12" - name: Install staging test dependencies - run: pip install pytest httpx + run: pip install pytest httpx sqlalchemy psycopg[binary] - name: Run staging smoke test - run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v + run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v env: API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }} + V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} promote-cloud-run-staging: name: Promote staging Cloud Run traffic @@ -332,6 +314,10 @@ jobs: environment: staging env: CLOUD_RUN_SERVICE: policyengine-api-staging + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} + V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} @@ -383,10 +369,216 @@ jobs: if: ${{ always() && (steps.promote.outcome == 'failure' || steps.stable_health.outcome == 'failure') }} run: exit 1 + exercise-phase10-staging: + name: Exercise Phase 10 activation, failure, retry, and rollback + runs-on: ubuntu-latest + timeout-minutes: 45 + needs: + - deploy-cloud-run-staging + - promote-cloud-run-staging + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + environment: staging + env: + DEPLOYMENT_ENVIRONMENT: staging + SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} + SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + POLICYENGINE_DB_READONLY_PASSWORD_SECRET: ${{ vars.POLICYENGINE_DB_READONLY_PASSWORD_SECRET }} + POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET: ${{ vars.POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET }} + PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET: ${{ vars.PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET }} + PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET: ${{ vars.PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET }} + ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} + ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} + ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} + ROUTE_IMPL_POLICY: ${{ vars.ROUTE_IMPL_POLICY }} + DB_READ_POLICY: ${{ vars.DB_READ_POLICY }} + DB_WRITE_POLICY: ${{ vars.DB_WRITE_POLICY }} + CLOUD_RUN_SERVICE: policyengine-api-staging + CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: staging + CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-staging-runtime-cache-url:latest + CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-staging-runtime-cache-ca:latest + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} + V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + PRODUCTION_V2_SUPABASE_PROJECT_REF: ${{ vars.PRODUCTION_V2_SUPABASE_PROJECT_REF }} + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} + V2_FAILURE_DATABASE_URL_SECRET_RESOURCE: ${{ vars.V2_FAILURE_DATABASE_URL_SECRET_RESOURCE }} + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} + CLOUD_RUN_MIN_INSTANCES: "0" + CLOUD_RUN_SERVICE_MIN_INSTANCES: "0" + CLOUD_RUN_MAX_INSTANCES: "1" + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install live exercise dependencies + run: pip install pytest httpx sqlalchemy psycopg[binary] pymysql + - name: Verify isolated Phase 10 exercise configuration + run: bash .github/scripts/validate_phase10_staging_exercise_env.sh + - name: Authenticate deployment identity + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }} + - name: Set up GCloud + uses: google-github-actions/setup-gcloud@v2 + - name: Install jq + run: sudo apt-get install -y jq + - name: Compute exercise revision tags + id: tags + run: | + echo "dual_write=p10-on-${GITHUB_RUN_NUMBER}-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + echo "failure=p10-f-${GITHUB_RUN_NUMBER}-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + - name: Deploy dual-write staging revision + run: bash .github/scripts/deploy_cloud_run_candidate.sh + env: + CLOUD_RUN_IMAGE_URI: ${{ needs.deploy-cloud-run-staging.outputs.image }} + CLOUD_RUN_TAG: ${{ steps.tags.outputs.dual_write }} + CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com + POLICYENGINE_DB_USER: ${{ vars.POLICYENGINE_DB_USER }} + POLICYENGINE_DB_NAME: ${{ vars.POLICYENGINE_DB_NAME }} + DB_WRITE_POLICY: dual_write + GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} + GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} + GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} + GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} + - name: Resolve exact dual-write staging revision + id: dual_write + run: bash .github/scripts/resolve_cloud_run_candidate_state.sh >> "$GITHUB_OUTPUT" + env: + CLOUD_RUN_TAG: ${{ steps.tags.outputs.dual_write }} + DB_WRITE_POLICY: dual_write + - name: Wait for dual-write revision health + run: bash .github/scripts/health_check.sh "${{ steps.dual_write.outputs.url }}/readiness-check" + - name: Promote dual-write staging revision + id: promote_dual_write + run: bash .github/scripts/set_cloud_run_revision.sh + env: + CLOUD_RUN_TARGET_REVISION: ${{ steps.dual_write.outputs.revision }} + CLOUD_RUN_EXPECTED_CURRENT_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.revision }} + - name: Wait for dual-write stable URL health + run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.stable_url }}/readiness-check" + - name: Deploy controlled-failure staging revision + run: bash .github/scripts/deploy_cloud_run_candidate.sh + env: + CLOUD_RUN_IMAGE_URI: ${{ needs.deploy-cloud-run-staging.outputs.image }} + CLOUD_RUN_TAG: ${{ steps.tags.outputs.failure }} + CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com + CLOUD_RUN_STARTUP_PROBE: httpGet.path=/health-check,httpGet.port=8080,initialDelaySeconds=0,periodSeconds=10,failureThreshold=24,timeoutSeconds=5 + POLICYENGINE_DB_USER: ${{ vars.POLICYENGINE_DB_USER }} + POLICYENGINE_DB_NAME: ${{ vars.POLICYENGINE_DB_NAME }} + DB_WRITE_POLICY: dual_write + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.V2_FAILURE_DATABASE_URL_SECRET_RESOURCE }} + GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} + GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} + GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} + GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} + - name: Resolve exact controlled-failure staging revision + id: failure + run: bash .github/scripts/resolve_cloud_run_candidate_state.sh >> "$GITHUB_OUTPUT" + env: + CLOUD_RUN_TAG: ${{ steps.tags.outputs.failure }} + DB_WRITE_POLICY: dual_write + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.V2_FAILURE_DATABASE_URL_SECRET_RESOURCE }} + - name: Authenticate database inspection identity + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }} + - name: Start staging Cloud SQL Auth Proxy + run: bash .github/scripts/start_cloud_sql_proxy.sh + - name: Run activation, failure, and retry checks + id: activation + continue-on-error: true + run: bash .github/scripts/run_phase10_staging_probe.sh activation + env: + API_BASE_URL: ${{ steps.dual_write.outputs.url }} + PHASE10_FAILURE_API_BASE_URL: ${{ steps.failure.outputs.url }} + PHASE10_STATE_PATH: ${{ runner.temp }}/phase10-staging-evidence.json + PHASE10_CLOUD_SQL_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.revision }} + PHASE10_DUAL_WRITE_REVISION: ${{ steps.dual_write.outputs.revision }} + PHASE10_FAILURE_REVISION: ${{ steps.failure.outputs.revision }} + RUN_PHASE10_STAGING_EXERCISE: "1" + STAGING_API_TEST_PROBE_ID: phase10-${{ github.run_id }}-${{ github.run_attempt }} + V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} + - name: Stop staging Cloud SQL Auth Proxy after activation + if: always() + run: bash .github/scripts/stop_cloud_sql_proxy.sh + - name: Reauthenticate deployment identity for rollback + if: always() && steps.promote_dual_write.outcome == 'success' + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }} + - name: Restore exact Cloud SQL-only staging revision + id: rollback + if: always() && steps.promote_dual_write.outcome == 'success' + continue-on-error: true + run: bash .github/scripts/set_cloud_run_revision.sh + env: + CLOUD_RUN_TARGET_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.revision }} + CLOUD_RUN_EXPECTED_CURRENT_REVISION: ${{ steps.dual_write.outputs.revision }} + - name: Wait for restored staging revision health + id: rollback_health + if: always() && steps.rollback.outcome == 'success' + continue-on-error: true + run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.stable_url }}/readiness-check" + - name: Authenticate database inspection identity after rollback + if: always() && steps.rollback_health.outcome == 'success' + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }} + - name: Restart staging Cloud SQL Auth Proxy + if: always() && steps.rollback_health.outcome == 'success' + run: bash .github/scripts/start_cloud_sql_proxy.sh + - name: Run Cloud SQL-only rollback checks + id: rollback_probe + if: always() && steps.activation.outcome == 'success' && steps.rollback_health.outcome == 'success' + continue-on-error: true + run: bash .github/scripts/run_phase10_staging_probe.sh rollback + env: + API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.stable_url }} + PHASE10_STATE_PATH: ${{ runner.temp }}/phase10-staging-evidence.json + PHASE10_CLOUD_SQL_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.revision }} + RUN_PHASE10_STAGING_EXERCISE: "1" + STAGING_API_TEST_PROBE_ID: phase10-${{ github.run_id }}-${{ github.run_attempt }} + V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} + - name: Stop staging Cloud SQL Auth Proxy after rollback + if: always() + run: bash .github/scripts/stop_cloud_sql_proxy.sh + - name: Retain non-secret Phase 10 staging evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: phase10-staging-evidence-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/phase10-staging-evidence.json + if-no-files-found: error + retention-days: 90 + - name: Fail after an incomplete Phase 10 staging exercise + if: >- + ${{ always() && ( + steps.activation.outcome != 'success' + || steps.rollback.outcome != 'success' + || steps.rollback_health.outcome != 'success' + || steps.rollback_probe.outcome != 'success' + ) }} + run: exit 1 + ensure-production-model-version-aligns-with-sim-api: name: Ensure production model version aligns with simulation API runs-on: ubuntu-latest - needs: promote-cloud-run-staging + needs: exercise-phase10-staging if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -402,9 +594,23 @@ jobs: - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh + migrate-v1-production-cloud-sql: + name: Upgrade production v1 Cloud SQL schema + needs: ensure-production-model-version-aligns-with-sim-api + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + uses: ./.github/workflows/migrate-v1-cloud-sql.yml + with: + deployment_environment: production + github_environment: production-database + secrets: inherit + seed-v2-production-database: name: Seed production v2 database - needs: ensure-production-model-version-aligns-with-sim-api + needs: + - ensure-production-model-version-aligns-with-sim-api + - migrate-v1-production-cloud-sql if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -422,9 +628,11 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine API') environment: production env: + DEPLOYMENT_ENVIRONMENT: production SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} @@ -435,9 +643,13 @@ jobs: CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: production CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-prod-runtime-cache-url:latest CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-prod-runtime-cache-ca:latest + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + PRODUCTION_V2_SUPABASE_PROJECT_REF: ${{ vars.PRODUCTION_V2_SUPABASE_PROJECT_REF }} V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} # Sized by the Stage 2 qualification and the PR 4 host cutover — rationale # and numbers in docs/migration/cloud-run-operations.md ("Runtime shape and # scaling"). Warm capacity is expressed service-level (--min); the diff --git a/.github/workflows/seed-v2-database.yml b/.github/workflows/seed-v2-database.yml index 52c3673e2..0b02da004 100644 --- a/.github/workflows/seed-v2-database.yml +++ b/.github/workflows/seed-v2-database.yml @@ -21,8 +21,10 @@ jobs: timeout-minutes: 30 environment: ${{ inputs.deployment_environment }} env: + DEPLOYMENT_ENVIRONMENT: ${{ inputs.deployment_environment }} V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + PRODUCTION_V2_SUPABASE_PROJECT_REF: ${{ vars.PRODUCTION_V2_SUPABASE_PROJECT_REF }} steps: - name: Checkout repo uses: actions/checkout@v4 @@ -34,6 +36,12 @@ jobs: uses: astral-sh/setup-uv@v6 - name: Install locked dependencies run: uv sync --frozen + - name: Verify the database environment is isolated + run: bash .github/scripts/validate_database_environment.sh supabase + - name: Qualify the pending policy migration + run: uv run python scripts/qualify_v2_policy_migration.py + env: + V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} - name: Upgrade and verify the v2 schema run: bash .github/scripts/migrate_v2_metadata_schema.sh env: diff --git a/docs/migration/stage-10-v2-policies.md b/docs/migration/stage-10-v2-policies.md index 0e7b160db..c9f33e541 100644 --- a/docs/migration/stage-10-v2-policies.md +++ b/docs/migration/stage-10-v2-policies.md @@ -27,6 +27,18 @@ integer identifiers remain in Cloud SQL throughout this stage. Continue only when the result reports zero policies, policy-owned parameter values, and user-policy associations requiring preservation. A nonzero count requires a separate preservation decision. +4. Staging must use independently writable copies of both production databases. + The staging Supabase project reference, Cloud SQL instance connection name, + v1 password secret, and v2 runtime URL secret must all differ from their + production values. The staging Cloud Run runtime identity must not have + access to either production database secret. + +The repository-level `PRODUCTION_*` variables record non-secret production +identities for comparison. The `staging`, `staging-database`, `production`, and +`production-database` GitHub environments supply their own target identities +and credential-resource names. The deployment and migration scripts stop +before connecting when the selected environment is missing or a staging value +equals its production counterpart. ## Schemas Before Traffic @@ -51,6 +63,15 @@ Run each Alembic `check` command against the same corresponding target and confirm no metadata/schema difference. The relevant generated revisions are `3d6e8f553ca5` for MySQL and `af34023a728f` for the current PostgreSQL head. +The release workflow applies and verifies both staging schemas before building +the staging candidate. It replaces, rather than appends to, the candidate's +Cloud SQL attachment and verifies the deployed revision's Cloud SQL attachment, +database identity environment variables, route implementation settings, and +database read/write settings. It then performs the complete activation, +controlled-failure, retry, and application-rollback exercise described below. +Production schema jobs are not eligible to run until that job has restored and +verified the exact Cloud SQL-only staging revision. + ## Activation The GitHub `staging` and `production` environments must define all three Phase @@ -96,6 +117,16 @@ DB_READ_POLICY=cloud_sql DB_WRITE_POLICY=dual_write ``` +The automated exercise deploys this selection as a distinct no-traffic +revision, verifies the exact immutable image and environment configuration, +and then assigns staging traffic to that revision. It also deploys a separate +no-traffic revision whose staging-only Secret Manager resource contains an +intentionally invalid password for the same staging Supabase project. That +revision uses `/health-check` only for process startup so the test can send a +real v1 write and verify the HTTP 503 response produced by an unavailable v2 +database. The invalid secret is accessible only to the staging runtime service +account and does not identify a production resource. + Under this selection, a core-policy mutation commits Cloud SQL first and then completes its policy transaction in Supabase. A saved-policy mutation commits its source row, incremented revision, and complete event in one Cloud SQL @@ -159,3 +190,11 @@ schema or the v2 schema. If a schema downgrade is separately approved, first disable native policy traffic and mirroring, verify that no pending Cloud SQL events or retained v2 data depend on the revisions, and run the reviewed Alembic downgrades against their confirmed database targets. + +The release workflow restores the exact preceding staging revision with +`DB_WRITE_POLICY=cloud_sql`, verifies the stable service URL, creates another +synthetic v1 policy, confirms that no v2 mapping was created for it, and reads a +policy that was committed to Supabase during activation. It uploads a +90-day-retained JSON artifact containing revision names, timestamps, selector +values, HTTP status summaries, synthetic record identifiers, and non-secret row +counts. The artifact never contains passwords or database URLs. diff --git a/policyengine_api/data/v2/policy_migration_qualification.py b/policyengine_api/data/v2/policy_migration_qualification.py index b5e295a92..e2224844e 100644 --- a/policyengine_api/data/v2/policy_migration_qualification.py +++ b/policyengine_api/data/v2/policy_migration_qualification.py @@ -2,13 +2,17 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +from collections.abc import Callable, Collection, Mapping from dataclasses import dataclass import json import sys from typing import Protocol -from sqlalchemy import Connection, Engine, func, select +from alembic.config import Config +from alembic.migration import MigrationContext +from alembic.script import ScriptDirectory +from sqlalchemy import Connection, Engine, func, inspect, select +from sqlalchemy.exc import NoInspectionAvailable from sqlalchemy.pool import NullPool from sqlmodel import create_engine @@ -20,6 +24,10 @@ ) +POLICY_MIGRATION_REVISION = "711ec2f0a5a5" +V2_ALEMBIC_CONFIG = "alembic-v2.ini" + + class ScalarExecutor(Protocol): """Minimal database interface required by the row-count queries.""" @@ -54,10 +62,12 @@ class PolicyMigrationQualification: environment: str project_ref: str counts: PolicyDataCounts + required: bool = True def as_dict(self) -> dict[str, object]: return { "outcome": "ok", + "qualification": "performed" if self.required else "not-required", "environment": self.environment, "project_ref": self.project_ref, "counts": self.counts.as_dict(), @@ -79,11 +89,21 @@ def __init__(self, counts: PolicyDataCounts) -> None: ) -def read_policy_data_counts(executor: ScalarExecutor) -> PolicyDataCounts: +def read_policy_data_counts( + executor: ScalarExecutor, + existing_tables: Collection[str] | None = None, +) -> PolicyDataCounts: """Count only policy-owned rows; canonical catalog values are excluded.""" + def table_exists(name: str) -> bool: + return existing_tables is None or name in existing_tables + return PolicyDataCounts( - policies=int(executor.scalar(select(func.count()).select_from(Policy)) or 0), + policies=( + int(executor.scalar(select(func.count()).select_from(Policy)) or 0) + if table_exists("policies") + else 0 + ), policy_parameter_values=int( executor.scalar( select(func.count()) @@ -91,9 +111,13 @@ def read_policy_data_counts(executor: ScalarExecutor) -> PolicyDataCounts: .where(ParameterValue.policy_id.is_not(None)) ) or 0 - ), - user_policies=int( - executor.scalar(select(func.count()).select_from(UserPolicy)) or 0 + ) + if table_exists("parameter_values") + else 0, + user_policies=( + int(executor.scalar(select(func.count()).select_from(UserPolicy)) or 0) + if table_exists("user_policies") + else 0 ), ) @@ -111,17 +135,35 @@ def build_qualification_engine(settings: V2DatabaseSettings) -> Engine: return create_engine(settings.connection.url, poolclass=NullPool) +def _read_and_require_counts(connection: Connection) -> PolicyDataCounts: + try: + existing_tables = inspect(connection).get_table_names(schema="public") + except NoInspectionAvailable: + existing_tables = None + counts = read_policy_data_counts(connection, existing_tables) + require_no_retained_policy_data(counts) + return counts + + def _qualify_connection(connection: Connection) -> PolicyDataCounts: transaction = connection.begin() try: connection.exec_driver_sql("SET TRANSACTION READ ONLY") - counts = read_policy_data_counts(connection) - require_no_retained_policy_data(counts) - return counts + return _read_and_require_counts(connection) finally: transaction.rollback() +def policy_migration_is_pending(connection: Connection) -> bool: + """Return whether the Phase 10 policy revision is in the upgrade path.""" + + current_heads = MigrationContext.configure(connection).get_current_heads() + revisions = ScriptDirectory.from_config( + Config(V2_ALEMBIC_CONFIG) + ).iterate_revisions("heads", current_heads) + return POLICY_MIGRATION_REVISION in {migration.revision for migration in revisions} + + def qualify_policy_migration_target( environ: Mapping[str, str] | None = None, *, @@ -145,6 +187,42 @@ def qualify_policy_migration_target( ) +def qualify_policy_migration_if_pending( + environ: Mapping[str, str] | None = None, + *, + engine_builder: Callable[[V2DatabaseSettings], Engine] = ( + build_qualification_engine + ), + pending_checker: Callable[[Connection], bool] = policy_migration_is_pending, +) -> PolicyMigrationQualification: + """Qualify predecessor rows only while the Phase 10 revision is pending.""" + + settings = load_v2_migration_database_settings(environ) + engine = engine_builder(settings) + try: + with engine.connect() as connection: + transaction = connection.begin() + try: + connection.exec_driver_sql("SET TRANSACTION READ ONLY") + if not pending_checker(connection): + return PolicyMigrationQualification( + environment=settings.target.environment, + project_ref=settings.target.project_ref, + counts=PolicyDataCounts(0, 0, 0), + required=False, + ) + counts = _read_and_require_counts(connection) + finally: + transaction.rollback() + finally: + engine.dispose() + return PolicyMigrationQualification( + environment=settings.target.environment, + project_ref=settings.target.project_ref, + counts=counts, + ) + + def _error_payload(error: Exception) -> dict[str, object]: safe_errors = (V2ConfigurationError, RetainedPolicyDataError) message = ( @@ -165,7 +243,7 @@ def main() -> int: """Run qualification and return a shell-compatible status.""" try: - evidence = qualify_policy_migration_target() + evidence = qualify_policy_migration_if_pending() except Exception as error: # noqa: BLE001 - command must emit safe evidence print(json.dumps(_error_payload(error), sort_keys=True), file=sys.stderr) return 1 diff --git a/scripts/qualify_v2_policy_migration.py b/scripts/qualify_v2_policy_migration.py index 32fc3ea5c..841ee20ea 100644 --- a/scripts/qualify_v2_policy_migration.py +++ b/scripts/qualify_v2_policy_migration.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Verify that a Supabase target has no retained v2 policy data.""" +"""Verify predecessor policy data before the Phase 10 revision is applied.""" from policyengine_api.data.v2.policy_migration_qualification import main diff --git a/tests/integration/test_live_phase10_staging.py b/tests/integration/test_live_phase10_staging.py new file mode 100644 index 000000000..630a1cc7d --- /dev/null +++ b/tests/integration/test_live_phase10_staging.py @@ -0,0 +1,498 @@ +"""Stateful activation and rollback checks for the isolated staging databases.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import json +import os +from pathlib import Path +from uuid import uuid4 + +import httpx +import pytest +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL +from sqlalchemy.pool import NullPool + + +V1_ONLY_COUNTRIES = ("ca", "ng", "il") +SYNTHETIC_PARAMETER = "gov.states.ut.tax.income.rate" + +pytestmark = pytest.mark.skipif( + os.environ.get("RUN_PHASE10_STAGING_EXERCISE") != "1", + reason="live Phase 10 staging exercise was not explicitly selected", +) + + +def _required_environment(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required for the live Phase 10 staging exercise") + return value + + +def _state_path() -> Path: + return Path(_required_environment("PHASE10_STATE_PATH")) + + +def _v1_engine(): + password = _required_environment("POLICYENGINE_DB_READONLY_PASSWORD") + return create_engine( + URL.create( + "mysql+pymysql", + username="policyengine_schema_reader", + password=password, + host="127.0.0.1", + port=3307, + database="policyengine", + ), + poolclass=NullPool, + ) + + +def _v2_engine(): + return create_engine( + _required_environment("V2_MIGRATION_DATABASE_URL"), + poolclass=NullPool, + ) + + +def _client(setting: str) -> httpx.Client: + return httpx.Client( + base_url=_required_environment(setting).rstrip("/"), + timeout=90, + follow_redirects=True, + ) + + +def _counts(v1_engine, v2_engine) -> dict[str, int]: + with v1_engine.connect() as connection: + v1_counts = { + "v1_policies": connection.scalar(text("SELECT COUNT(*) FROM policy")), + "v1_user_policies": connection.scalar( + text("SELECT COUNT(*) FROM user_policies") + ), + "v1_pending_mirror_events": connection.scalar( + text( + "SELECT COUNT(*) FROM user_policy_mirror_events " + "WHERE processed_at IS NULL" + ) + ), + } + with v2_engine.connect() as connection: + v2_counts = { + "v2_policies": connection.scalar(text("SELECT COUNT(*) FROM policies")), + "v2_user_policies": connection.scalar( + text("SELECT COUNT(*) FROM user_policies") + ), + "v2_policy_mappings": connection.scalar( + text("SELECT COUNT(*) FROM legacy_policy_mappings") + ), + "v2_user_policy_mappings": connection.scalar( + text("SELECT COUNT(*) FROM legacy_user_policy_mappings") + ), + } + return {key: int(value) for key, value in {**v1_counts, **v2_counts}.items()} + + +def _policy_payload(label: str, value: float) -> dict[str, object]: + return { + "label": label, + "data": { + SYNTHETIC_PARAMETER: { + "2026-01-01.2100-12-31": value, + } + }, + } + + +def _saved_policy_payload( + *, probe_id: str, reform_id: int, reform_label: str +) -> dict[str, object]: + return { + "reform_id": reform_id, + "reform_label": reform_label, + "baseline_id": reform_id, + "baseline_label": "Synthetic staging baseline", + "user_id": f"phase10-staging-{probe_id}", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 1, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": "phase10-staging", + } + + +def _assert_v1_policy_identity(response: httpx.Response, expected_id: int) -> None: + assert response.status_code == 200, response.text[:500] + item = response.json()["result"] + assert item["id"] == expected_id + assert isinstance(item["id"], int) + assert "uuid" not in item + assert "v2_policy_id" not in item + + +def test_live_phase10_activation_failure_and_retry(integration_probe_id: str) -> None: + """Activate immediate mirroring and prove deterministic retry semantics.""" + + probe_id = integration_probe_id.replace("/", "-") + unique_value = 0.04 + (int(uuid4().hex[:6], 16) % 5000) / 1_000_000 + v1_engine = _v1_engine() + v2_engine = _v2_engine() + evidence: dict[str, object] = { + "environment": "staging", + "started_at": datetime.now(timezone.utc).isoformat(), + "probe_id": probe_id, + "revisions": { + "cloud_sql_only": _required_environment("PHASE10_CLOUD_SQL_REVISION"), + "dual_write": _required_environment("PHASE10_DUAL_WRITE_REVISION"), + "controlled_failure": _required_environment("PHASE10_FAILURE_REVISION"), + }, + "activation_selectors": { + "ROUTE_IMPL_POLICY": "fastapi_native", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "dual_write", + }, + "status_summary": {}, + } + try: + evidence["counts_before"] = _counts(v1_engine, v2_engine) + first_label = f"Phase 10 failure retry {probe_id}" + first_payload = _policy_payload(first_label, unique_value) + + with _client("PHASE10_FAILURE_API_BASE_URL") as failure_client: + failed = failure_client.post("/us/policy", json=first_payload) + assert failed.status_code == 503, failed.text[:500] + evidence["status_summary"]["controlled_supabase_failure"] = 503 + + with _client("API_BASE_URL") as active_client: + retried = active_client.post("/us/policy", json=first_payload) + assert retried.status_code == 200, retried.text[:500] + first_legacy_id = retried.json()["result"]["policy_id"] + assert isinstance(first_legacy_id, int) + evidence["status_summary"]["policy_retry"] = 200 + + _assert_v1_policy_identity( + active_client.get(f"/us/policy/{first_legacy_id}"), + first_legacy_id, + ) + + with v2_engine.connect() as connection: + first_v2_id = connection.scalar( + text( + "SELECT policy_id FROM legacy_policy_mappings " + "WHERE country_id = 'us' AND legacy_policy_id = :legacy_id" + ), + {"legacy_id": first_legacy_id}, + ) + assert first_v2_id is not None + + equivalent_payload = _policy_payload( + f"Phase 10 equivalent label {probe_id}", + unique_value, + ) + equivalent = active_client.post("/us/policy", json=equivalent_payload) + assert equivalent.status_code == 201, equivalent.text[:500] + equivalent_legacy_id = equivalent.json()["result"]["policy_id"] + with v2_engine.connect() as connection: + equivalent_v2_id = connection.scalar( + text( + "SELECT policy_id FROM legacy_policy_mappings " + "WHERE country_id = 'us' AND legacy_policy_id = :legacy_id" + ), + {"legacy_id": equivalent_legacy_id}, + ) + assert equivalent_v2_id == first_v2_id + evidence["status_summary"]["equivalent_content_deduplication"] = 201 + + saved_label = f"Phase 10 saved failure {probe_id}" + saved_payload = _saved_policy_payload( + probe_id=probe_id, + reform_id=first_legacy_id, + reform_label=saved_label, + ) + + with _client("PHASE10_FAILURE_API_BASE_URL") as failure_client: + failed_saved = failure_client.post("/us/user-policy", json=saved_payload) + assert failed_saved.status_code == 503, failed_saved.text[:500] + evidence["status_summary"]["saved_policy_controlled_failure"] = 503 + + with _client("API_BASE_URL") as active_client: + retried_saved = active_client.post("/us/user-policy", json=saved_payload) + assert retried_saved.status_code == 200, retried_saved.text[:500] + saved_policy_id = retried_saved.json()["result"]["id"] + + with v1_engine.connect() as connection: + events = connection.execute( + text( + "SELECT source_revision, processed_at " + "FROM user_policy_mirror_events " + "WHERE country_id = 'us' " + "AND legacy_user_policy_id = :legacy_id " + "ORDER BY source_revision" + ), + {"legacy_id": saved_policy_id}, + ).all() + assert [event.source_revision for event in events] == [1, 2] + assert all(event.processed_at is not None for event in events) + assert events[0].processed_at <= events[1].processed_at + + with v2_engine.connect() as connection: + saved_mapping = connection.execute( + text( + "SELECT m.user_policy_id, m.last_applied_source_revision, " + "m.fingerprint_sha256, a.user_id, a.name, a.description " + "FROM legacy_user_policy_mappings m " + "JOIN user_policies a ON a.id = m.user_policy_id " + "WHERE m.country_id = 'us' " + "AND m.legacy_user_policy_id = :legacy_id" + ), + {"legacy_id": saved_policy_id}, + ).one() + mapped_legacy_user_id = connection.scalar( + text( + "SELECT legacy_user_id FROM legacy_user_mappings " + "WHERE user_id = :user_id" + ), + {"user_id": saved_mapping.user_id}, + ) + assert saved_mapping.last_applied_source_revision == 2 + assert mapped_legacy_user_id == saved_payload["user_id"] + assert saved_mapping.name == saved_label + assert saved_mapping.description is None + + renamed_label = f"Phase 10 saved renamed {probe_id}" + renamed = active_client.put( + "/us/user-policy", + json={"id": saved_policy_id, "reform_label": renamed_label}, + ) + assert renamed.status_code == 200, renamed.text[:500] + with v2_engine.connect() as connection: + renamed_mapping = connection.execute( + text( + "SELECT m.last_applied_source_revision, " + "m.fingerprint_sha256, a.name, a.description " + "FROM legacy_user_policy_mappings m " + "JOIN user_policies a ON a.id = m.user_policy_id " + "WHERE m.country_id = 'us' " + "AND m.legacy_user_policy_id = :legacy_id" + ), + {"legacy_id": saved_policy_id}, + ).one() + assert renamed_mapping.last_applied_source_revision == 3 + assert renamed_mapping.name == renamed_label + assert renamed_mapping.description is None + + v1_only_update = active_client.put( + "/us/user-policy", + json={"id": saved_policy_id, "number_of_provisions": 2}, + ) + assert v1_only_update.status_code == 200, v1_only_update.text[:500] + with v2_engine.connect() as connection: + v1_only_mapping = connection.execute( + text( + "SELECT m.last_applied_source_revision, " + "m.fingerprint_sha256, a.name, a.description " + "FROM legacy_user_policy_mappings m " + "JOIN user_policies a ON a.id = m.user_policy_id " + "WHERE m.country_id = 'us' " + "AND m.legacy_user_policy_id = :legacy_id" + ), + {"legacy_id": saved_policy_id}, + ).one() + assert v1_only_mapping.last_applied_source_revision == 4 + assert ( + v1_only_mapping.fingerprint_sha256 != renamed_mapping.fingerprint_sha256 + ) + assert v1_only_mapping.name == renamed_label + assert v1_only_mapping.description is None + + uk_parameters = active_client.get( + "/v2/parameters", + params={"country_id": "uk", "limit": 1}, + ) + assert uk_parameters.status_code == 200, uk_parameters.text[:500] + uk_parameter_name = uk_parameters.json()["result"]["items"][0]["name"] + uk_policy = active_client.post( + "/uk/policy", + json={ + "label": f"Phase 10 UK {probe_id}", + "data": { + uk_parameter_name: { + "2026-01-01.2100-12-31": unique_value, + } + }, + }, + ) + assert uk_policy.status_code == 201, uk_policy.text[:500] + uk_legacy_id = uk_policy.json()["result"]["policy_id"] + with v2_engine.connect() as connection: + uk_mapping_count = connection.scalar( + text( + "SELECT COUNT(*) FROM legacy_policy_mappings " + "WHERE country_id = 'uk' AND legacy_policy_id = :legacy_id" + ), + {"legacy_id": uk_legacy_id}, + ) + assert uk_mapping_count == 1 + + with _client("PHASE10_FAILURE_API_BASE_URL") as failure_client: + for country_id in V1_ONLY_COUNTRIES: + unsupported_policy = failure_client.post( + f"/{country_id}/policy", + json=_policy_payload( + f"Phase 10 {country_id} v1-only {probe_id}", + unique_value, + ), + ) + assert unsupported_policy.status_code == 201, unsupported_policy.text[ + :500 + ] + unsupported_id = unsupported_policy.json()["result"]["policy_id"] + _assert_v1_policy_identity( + failure_client.get(f"/{country_id}/policy/{unsupported_id}"), + unsupported_id, + ) + unsupported_saved = failure_client.post( + f"/{country_id}/user-policy", + json={ + **_saved_policy_payload( + probe_id=f"{country_id}-{probe_id}", + reform_id=unsupported_id, + reform_label=f"Phase 10 {country_id} saved {probe_id}", + ), + "geography": country_id, + }, + ) + assert unsupported_saved.status_code == 201, unsupported_saved.text[ + :500 + ] + unsupported_saved_id = unsupported_saved.json()["result"]["id"] + with v2_engine.connect() as connection: + mapping_count = connection.scalar( + text( + "SELECT COUNT(*) FROM legacy_policy_mappings " + "WHERE legacy_policy_id = :legacy_id" + ), + {"legacy_id": unsupported_id}, + ) + assert mapping_count == 0 + with v1_engine.connect() as connection: + event_count = connection.scalar( + text( + "SELECT COUNT(*) FROM user_policy_mirror_events " + "WHERE country_id = :country_id " + "AND legacy_user_policy_id = :legacy_id" + ), + { + "country_id": country_id, + "legacy_id": unsupported_saved_id, + }, + ) + assert event_count == 0 + + with v1_engine.connect() as connection: + final_events = connection.execute( + text( + "SELECT source_revision, processed_at " + "FROM user_policy_mirror_events " + "WHERE country_id = 'us' " + "AND legacy_user_policy_id = :legacy_id " + "ORDER BY source_revision" + ), + {"legacy_id": saved_policy_id}, + ).all() + assert [event.source_revision for event in final_events] == [1, 2, 3, 4] + assert all(event.processed_at is not None for event in final_events) + + evidence["status_summary"].update( + { + "saved_policy_retry": 200, + "saved_policy_label_update": 200, + "saved_policy_v1_only_update": 200, + "uk_policy_mirror": 201, + "v1_only_country_writes": 201, + } + ) + evidence["synthetic_ids"] = { + "legacy_policy_id": first_legacy_id, + "equivalent_legacy_policy_id": equivalent_legacy_id, + "v2_policy_id": str(first_v2_id), + "legacy_user_policy_id": saved_policy_id, + "v2_user_policy_id": str(saved_mapping.user_policy_id), + } + evidence["counts_after_activation"] = _counts(v1_engine, v2_engine) + _state_path().write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + finally: + v1_engine.dispose() + v2_engine.dispose() + + +def test_live_phase10_cloud_sql_only_rollback(integration_probe_id: str) -> None: + """Verify rollback removes the v2 dependency without removing v2 state.""" + + evidence = json.loads(_state_path().read_text(encoding="utf-8")) + v1_engine = _v1_engine() + v2_engine = _v2_engine() + try: + unique_value = 0.05 + (int(uuid4().hex[:6], 16) % 5000) / 1_000_000 + rollback_label = f"Phase 10 rollback {integration_probe_id}" + with _client("API_BASE_URL") as rollback_client: + rollback_created = rollback_client.post( + "/us/policy", + json=_policy_payload(rollback_label, unique_value), + ) + assert rollback_created.status_code == 201, rollback_created.text[:500] + rollback_legacy_id = rollback_created.json()["result"]["policy_id"] + _assert_v1_policy_identity( + rollback_client.get(f"/us/policy/{rollback_legacy_id}"), + rollback_legacy_id, + ) + + with v2_engine.connect() as connection: + rollback_mapping_count = connection.scalar( + text( + "SELECT COUNT(*) FROM legacy_policy_mappings " + "WHERE country_id = 'us' AND legacy_policy_id = :legacy_id" + ), + {"legacy_id": rollback_legacy_id}, + ) + assert rollback_mapping_count == 0 + + retained_v2_policy_id = evidence["synthetic_ids"]["v2_policy_id"] + retained = rollback_client.get( + f"/v2/policies/{retained_v2_policy_id}", + params={"country_id": "us"}, + ) + assert retained.status_code == 200, retained.text[:500] + assert retained.json()["result"]["item"]["id"] == retained_v2_policy_id + + evidence["rollback"] = { + "completed_at": datetime.now(timezone.utc).isoformat(), + "revision": _required_environment("PHASE10_CLOUD_SQL_REVISION"), + "selectors": { + "ROUTE_IMPL_POLICY": "fastapi_native", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", + }, + "status_summary": { + "v1_cloud_sql_only_policy_create": 201, + "retained_v2_policy_read": 200, + }, + "synthetic_legacy_policy_id": rollback_legacy_id, + } + evidence["counts_after_rollback"] = _counts(v1_engine, v2_engine) + _state_path().write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + finally: + v1_engine.dispose() + v2_engine.dispose() diff --git a/tests/integration/test_live_v2_policies.py b/tests/integration/test_live_v2_policies.py new file mode 100644 index 000000000..e1584c17a --- /dev/null +++ b/tests/integration/test_live_v2_policies.py @@ -0,0 +1,188 @@ +"""Write/read lifecycle checks for deployed native v2 policy resources.""" + +from __future__ import annotations + +import os +from uuid import uuid4 + +from sqlalchemy import create_engine, text +from sqlalchemy.pool import NullPool + + +def _v2_database_url() -> str: + value = os.environ.get("V2_MIGRATION_DATABASE_URL") + if not value: + raise RuntimeError( + "V2_MIGRATION_DATABASE_URL is required for the live v2 probe" + ) + return value + + +def _one_catalog_item(api_client, resource: str, *, country_id: str) -> dict: + response = api_client.get( + f"/v2/{resource}", + params={"country_id": country_id, "limit": 1}, + ) + assert response.status_code == 200, response.text[:500] + items = response.json()["result"]["items"] + assert len(items) == 1 + return items[0] + + +def test_live_native_policy_and_user_policy_lifecycle( + api_client, + integration_probe_id: str, +) -> None: + """Exercise every Phase 10 native route against the staging database.""" + + country_id = "us" + model = _one_catalog_item( + api_client, + "tax-benefit-models", + country_id=country_id, + ) + parameter = _one_catalog_item( + api_client, + "parameters", + country_id=country_id, + ) + user_id = uuid4() + engine = create_engine(_v2_database_url(), poolclass=NullPool) + try: + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO users (id, primary_country) VALUES (:id, :country_id)" + ), + {"id": user_id, "country_id": country_id}, + ) + existing_policy_count = connection.scalar( + text( + "SELECT COUNT(*) FROM policies " + "WHERE country_id = :country_id " + "AND tax_benefit_model_id = :model_id" + ), + {"country_id": country_id, "model_id": model["id"]}, + ) + + policy_body = { + "country_id": country_id, + "tax_benefit_model_id": model["id"], + "parameter_values": [ + { + "parameter_id": parameter["id"], + "value": f"phase10-native-{integration_probe_id}", + "start_date": "2026-01-01T00:00:00Z", + } + ], + } + created = api_client.post( + "/v2/policies", + params={"country_id": country_id}, + json=policy_body, + ) + assert created.status_code == 201, created.text[:500] + created_item = created.json()["result"]["item"] + policy_id = created_item["id"] + assert created_item["country_id"] == country_id + assert created_item["tax_benefit_model_id"] == model["id"] + assert created_item["parameter_values"][0]["parameter_id"] == parameter["id"] + assert ( + created_item["parameter_values"][0]["parameter_name"] == parameter["name"] + ) + assert created_item["created_at"] + assert created_item["updated_at"] + + repeated = api_client.post( + "/v2/policies", + params={"country_id": country_id}, + json=policy_body, + ) + assert repeated.status_code == 200, repeated.text[:500] + assert repeated.json()["result"]["item"]["id"] == policy_id + + detail = api_client.get( + f"/v2/policies/{policy_id}", + params={"country_id": country_id}, + ) + assert detail.status_code == 200, detail.text[:500] + assert detail.json()["result"]["item"] == created_item + + collection = api_client.get( + "/v2/policies", + params={ + "country_id": country_id, + "tax_benefit_model_id": model["id"], + "offset": existing_policy_count, + "limit": 1, + }, + ) + assert collection.status_code == 200, collection.text[:500] + assert policy_id in { + item["id"] for item in collection.json()["result"]["items"] + } + + association_body = { + "country_id": country_id, + "user_id": str(user_id), + "policy_id": policy_id, + "name": f"Phase 10 native {integration_probe_id}", + "description": "Synthetic staging lifecycle record", + } + association = api_client.post( + "/v2/user-policies", + params={"country_id": country_id}, + json=association_body, + ) + assert association.status_code == 201, association.text[:500] + association_item = association.json()["result"]["item"] + association_id = association_item["id"] + + association_detail = api_client.get( + f"/v2/user-policies/{association_id}", + params={"country_id": country_id}, + ) + assert association_detail.status_code == 200, association_detail.text[:500] + assert association_detail.json()["result"]["item"] == association_item + + association_collection = api_client.get( + "/v2/user-policies", + params={ + "country_id": country_id, + "user_id": str(user_id), + "policy_id": policy_id, + }, + ) + assert association_collection.status_code == 200 + assert [ + item["id"] for item in association_collection.json()["result"]["items"] + ] == [association_id] + + updated_name = f"Phase 10 renamed {integration_probe_id}" + patched = api_client.patch( + f"/v2/user-policies/{association_id}", + params={"country_id": country_id}, + json={"name": updated_name}, + ) + assert patched.status_code == 200, patched.text[:500] + patched_item = patched.json()["result"]["item"] + assert patched_item["name"] == updated_name + assert patched_item["description"] == association_body["description"] + + deleted = api_client.delete( + f"/v2/user-policies/{association_id}", + params={"country_id": country_id}, + ) + assert deleted.status_code == 204, deleted.text[:500] + absent = api_client.get( + f"/v2/user-policies/{association_id}", + params={"country_id": country_id}, + ) + assert absent.status_code == 404, absent.text[:500] + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM users WHERE id = :id"), + {"id": user_id}, + ) + engine.dispose() diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 6e0409686..c624c686f 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -84,7 +84,7 @@ def test_push_requires_schema_and_v2_integration_checks_before_versioning(): versioning_job = workflow[workflow.index(" versioning:") :] versioning_job = versioning_job[: versioning_job.index("\n publish-git-tag:")] tag_job = workflow[workflow.index(" publish-git-tag:") :] - tag_job = tag_job[: tag_job.index("\n migrate-v1-cloud-sql:")] + tag_job = tag_job[: tag_job.index("\n migrate-v1-staging-cloud-sql:")] assert "lint:" in workflow assert "alembic-v1-check:" in workflow @@ -106,16 +106,12 @@ def test_push_requires_schema_and_v2_integration_checks_before_versioning(): def test_release_migration_uses_the_installed_python_environment(): - workflow = _workflow("push.yml") - migration_job = workflow[workflow.index(" migrate-v1-cloud-sql:") :] - migration_job = migration_job[ - : migration_job.index("\n deploy-cloud-run-staging:") - ] + workflow = _workflow("migrate-v1-cloud-sql.yml") orchestration_script = ( REPO / ".github" / "scripts" / "migrate_v1_cloud_sql.sh" ).read_text(encoding="utf-8") - assert "bash .github/scripts/migrate_v1_cloud_sql.sh" in migration_job + assert "bash .github/scripts/migrate_v1_cloud_sql.sh" in workflow assert "python scripts/v1_database_migration.py" in orchestration_script assert "uv run" not in orchestration_script @@ -201,8 +197,8 @@ def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy(): REPO / ".github" / "scripts" / "migrate_v1_cloud_sql.sh" ).read_text(encoding="utf-8") - assert "migrate-v1-cloud-sql:" in workflow - assert "environment: production-database" in workflow + assert "migrate-v1-staging-cloud-sql:" in workflow + assert "migrate-v1-production-cloud-sql:" in workflow assert "--mode state" in orchestration_script assert "--mode upgrade" in orchestration_script assert "--mode verify-head" in orchestration_script @@ -213,7 +209,7 @@ def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy(): cloud_run_job = cloud_run_job[ : cloud_run_job.index("\n integration-tests-staging-cloud-run:") ] - assert "migrate-v1-cloud-sql" in cloud_run_job + assert "migrate-v1-staging-cloud-sql" in cloud_run_job assert "make test" in cloud_run_job assert cloud_run_job.index("make test") < cloud_run_job.index( 'uses: "google-github-actions/auth@v2"' @@ -224,11 +220,7 @@ def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy(): def test_cloud_sql_workflow_uses_oidc_and_separate_database_credentials(): - workflow = _workflow("push.yml") - migration_job = workflow[workflow.index(" migrate-v1-cloud-sql:") :] - migration_job = migration_job[ - : migration_job.index("\n deploy-cloud-run-staging:") - ] + migration_job = _workflow("migrate-v1-cloud-sql.yml") orchestration_script = ( REPO / ".github" / "scripts" / "migrate_v1_cloud_sql.sh" ).read_text(encoding="utf-8") @@ -246,6 +238,37 @@ def test_cloud_sql_workflow_uses_oidc_and_separate_database_credentials(): ) +def test_database_environment_validation_rejects_production_migration_credentials_in_staging(): + result = subprocess.run( + ["bash", ".github/scripts/validate_database_environment.sh", "cloud-sql"], + cwd=REPO, + env={ + **os.environ, + "DEPLOYMENT_ENVIRONMENT": "staging", + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + "project:region:staging-instance" + ), + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + "project:region:production-instance" + ), + "POLICYENGINE_DB_READONLY_PASSWORD_SECRET": "production-readonly", + "POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": "staging-migration", + "PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "production-readonly" + ), + "PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "production-migration" + ), + }, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 1 + assert "must be distinct from production" in result.stderr + + def _write_fake_migration_commands(tmp_path: Path) -> tuple[Path, Path]: bin_path = tmp_path / "bin" bin_path.mkdir() @@ -292,6 +315,22 @@ def _run_migration_orchestrator(tmp_path: Path, database_state: str): "GCLOUD_CALLS": str(gcloud_calls), "PYTHON_CALLS": str(python_calls), "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": "project:region:instance", + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + "project:region:instance" + ), + "DEPLOYMENT_ENVIRONMENT": "production", + "POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "policyengine-api-prod-db-readonly-password" + ), + "POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "policyengine-api-prod-db-migration-password" + ), + "PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "policyengine-api-prod-db-readonly-password" + ), + "PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "policyengine-api-prod-db-migration-password" + ), "PATH": f"{bin_path}:{os.environ['PATH']}", }, capture_output=True, diff --git a/tests/unit/test_app_engine_decommission.py b/tests/unit/test_app_engine_decommission.py index 7d738f38d..1a5896f0f 100644 --- a/tests/unit/test_app_engine_decommission.py +++ b/tests/unit/test_app_engine_decommission.py @@ -69,6 +69,7 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: "integration-tests-staging-cloud-run", ) staging_promotion = _job_block(workflow, "promote-cloud-run-staging") + staging_phase10_exercise = _job_block(workflow, "exercise-phase10-staging") production_check = _job_block( workflow, "ensure-production-model-version-aligns-with-sim-api", @@ -78,7 +79,7 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: docker_publish = _job_block(workflow, "docker") assert " release-tests:" not in workflow - assert "migrate-v1-cloud-sql" in staging_seed + assert "migrate-v1-staging-cloud-sql" in staging_seed assert "- seed-v2-staging-database" in staging_deploy assert "make test" in staging_deploy assert staging_deploy.index("make test") < staging_deploy.index( @@ -89,10 +90,9 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: ) assert "needs: deploy-cloud-run-staging" in staging_integration assert "- integration-tests-staging-cloud-run" in staging_promotion - assert "needs: promote-cloud-run-staging" in production_check - assert ( - "needs: ensure-production-model-version-aligns-with-sim-api" in production_seed - ) + assert "- promote-cloud-run-staging" in staging_phase10_exercise + assert "needs: exercise-phase10-staging" in production_check + assert "migrate-v1-production-cloud-sql" in production_seed assert "needs: seed-v2-production-database" in production_deploy assert "needs: deploy-cloud-run-candidate" in docker_publish diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 2efd748ca..f16174f08 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -74,7 +74,21 @@ def _v2_target_env() -> dict[str, str]: def _required_runtime_env() -> dict[str, str]: return { + "DEPLOYMENT_ENVIRONMENT": "production", "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": PRODUCTION_CLOUD_SQL_INSTANCE, + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + PRODUCTION_CLOUD_SQL_INSTANCE + ), + "PRODUCTION_V2_SUPABASE_PROJECT_REF": TEST_V2_PROJECT_REF, + "PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": ( + TEST_V2_RUNTIME_SECRET_RESOURCE + ), + "PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET": ( + "policyengine-api-prod-db-password:latest" + ), + "CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET": ( + "policyengine-api-prod-db-password:latest" + ), "POLICYENGINE_DB_PASSWORD": "raw-db-secret-value", "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": ("raw-github-secret-value"), "OPENAI_API_KEY": "raw-openai-secret-value", @@ -122,7 +136,16 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: "ROUTE_IMPL_POLICY": "flask_fallback", "DB_READ_POLICY": "cloud_sql", "DB_WRITE_POLICY": "cloud_sql", + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + PRODUCTION_CLOUD_SQL_INSTANCE + ), + "V2_SUPABASE_PROJECT_REF": TEST_V2_PROJECT_REF, + "V2_SUPABASE_ENVIRONMENT": TEST_V2_ENVIRONMENT, + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": ( + TEST_V2_RUNTIME_SECRET_RESOURCE + ), }, + "candidate_cloud_sql": PRODUCTION_CLOUD_SQL_INSTANCE, "updates": [], } ), @@ -161,7 +184,12 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: json.dumps( { "metadata": { - "labels": {"serving.knative.dev/service": revision_service} + "labels": {"serving.knative.dev/service": revision_service}, + "annotations": { + "run.googleapis.com/cloudsql-instances": state.get( + "candidate_cloud_sql", "" + ) + }, }, "spec": { "containers": [ @@ -511,12 +539,12 @@ def test_production_gunicorn_workers_do_not_inherit_database_pools(): def test_validate_cloud_run_deploy_env_requires_selector_environment_variable(): + env = _script_env(**_required_runtime_env()) + env.pop("SIM_ENTRYPOINT") + env.pop("SIMULATION_ENTRYPOINT_URL") result = _run_script( ".github/scripts/validate_cloud_run_deploy_env.sh", - _script_env( - OLD_SIMULATION_GATEWAY_URL="https://old-gateway.example.test", - **_gateway_auth_env(), - ), + env, ) assert result.returncode == 1 @@ -536,6 +564,20 @@ def test_validate_cloud_run_deploy_env_accepts_direct_mode_from_environment(): DB_READ_POLICY="cloud_sql", DB_WRITE_POLICY="cloud_sql", POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, + DEPLOYMENT_ENVIRONMENT="production", + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=( + PRODUCTION_CLOUD_SQL_INSTANCE + ), + PRODUCTION_V2_SUPABASE_PROJECT_REF=TEST_V2_PROJECT_REF, + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE=( + TEST_V2_RUNTIME_SECRET_RESOURCE + ), + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), **_v2_target_env(), **_gateway_auth_env(), ), @@ -651,6 +693,20 @@ def test_validate_cloud_run_deploy_env_requires_only_selected_url( DB_READ_POLICY="cloud_sql", DB_WRITE_POLICY="cloud_sql", POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, + DEPLOYMENT_ENVIRONMENT="production", + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=( + PRODUCTION_CLOUD_SQL_INSTANCE + ), + PRODUCTION_V2_SUPABASE_PROJECT_REF=TEST_V2_PROJECT_REF, + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE=( + TEST_V2_RUNTIME_SECRET_RESOURCE + ), + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), **_v2_target_env(), **_gateway_auth_env(), ) @@ -732,6 +788,129 @@ def test_deployment_validation_requires_database_instance_connection_name(): assert "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME" in result.stderr +def _staging_runtime_env() -> dict[str, str]: + return { + **_required_runtime_env(), + "DEPLOYMENT_ENVIRONMENT": "staging", + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + "policyengine-api:us-central1:policyengine-api-data-staging" + ), + "CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET": ( + "policyengine-api-staging-db-password:latest" + ), + "V2_SUPABASE_PROJECT_REF": "z" * 20, + "V2_SUPABASE_ENVIRONMENT": "staging", + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": ( + "projects/test-project/secrets/v2-staging-runtime-url/versions/latest" + ), + } + + +def _phase10_staging_exercise_env() -> dict[str, str]: + return { + **_staging_runtime_env(), + "ROUTE_IMPL_POLICY": "fastapi_native", + "POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "policyengine-api-staging-db-readonly-password" + ), + "POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "policyengine-api-staging-db-migration-password" + ), + "PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "policyengine-api-prod-db-readonly-password" + ), + "PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "policyengine-api-prod-db-migration-password" + ), + "V2_FAILURE_DATABASE_URL_SECRET_RESOURCE": ( + "projects/test-project/secrets/v2-staging-unavailable-url/versions/latest" + ), + } + + +def test_deployment_validation_accepts_distinct_staging_database_targets(): + result = _run_script( + ".github/scripts/validate_cloud_run_deploy_env.sh", + _script_env(**_staging_runtime_env()), + ) + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("setting", "production_setting"), + [ + ( + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME", + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME", + ), + ("V2_SUPABASE_PROJECT_REF", "PRODUCTION_V2_SUPABASE_PROJECT_REF"), + ( + "CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET", + "PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET", + ), + ( + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", + "PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", + ), + ], +) +def test_deployment_validation_rejects_shared_staging_database_targets( + setting, + production_setting, +): + env = _staging_runtime_env() + env[setting] = env[production_setting] + + result = _run_script( + ".github/scripts/validate_cloud_run_deploy_env.sh", + _script_env(**env), + ) + + assert result.returncode == 1 + assert "distinct from production" in result.stderr + + +def test_phase10_staging_exercise_requires_isolated_activation_configuration(): + result = _run_script( + ".github/scripts/validate_phase10_staging_exercise_env.sh", + _script_env(**_phase10_staging_exercise_env()), + ) + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("setting", "invalid_value", "message"), + [ + ("DEPLOYMENT_ENVIRONMENT", "production", "only against staging"), + ("ROUTE_IMPL_POLICY", "flask_fallback", "must be fastapi_native"), + ("DB_READ_POLICY", "supabase", "must remain cloud_sql"), + ("DB_WRITE_POLICY", "dual_write", "must begin with"), + ( + "V2_FAILURE_DATABASE_URL_SECRET_RESOURCE", + "projects/test-project/secrets/v2-staging-runtime-url/versions/latest", + "must differ from the valid staging secret", + ), + ], +) +def test_phase10_staging_exercise_rejects_unsafe_configuration( + setting, + invalid_value, + message, +): + env = _phase10_staging_exercise_env() + env[setting] = invalid_value + + result = _run_script( + ".github/scripts/validate_phase10_staging_exercise_env.sh", + _script_env(**env), + ) + + assert result.returncode == 1 + assert message in result.stderr + + def test_build_cloud_run_image_dry_run_uses_cloud_run_dockerfile(): dockerignore = REPO / "gcp/cloud_run/Dockerfile.dockerignore" @@ -774,7 +953,7 @@ def test_deploy_cloud_run_candidate_dry_run_never_shifts_traffic(): f"--service-account {DEDICATED_CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" in result.stdout ) - assert f"--add-cloudsql-instances {PRODUCTION_CLOUD_SQL_INSTANCE}" in result.stdout + assert f"--set-cloudsql-instances {PRODUCTION_CLOUD_SQL_INSTANCE}" in result.stdout assert ( f"POLICYENGINE_DB_INSTANCE_CONNECTION_NAME={PRODUCTION_CLOUD_SQL_INSTANCE}" in result.stdout @@ -842,6 +1021,7 @@ def test_deploy_cloud_run_candidate_uses_configured_database_instance(): env = { **_required_runtime_env(), "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": configured_instance, + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": configured_instance, } result = _run_script( ".github/scripts/deploy_cloud_run_candidate.sh", @@ -853,7 +1033,7 @@ def test_deploy_cloud_run_candidate_uses_configured_database_instance(): ) assert result.returncode == 0, result.stderr - assert f"--add-cloudsql-instances {configured_instance}" in result.stdout + assert f"--set-cloudsql-instances {configured_instance}" in result.stdout assert ( f"POLICYENGINE_DB_INSTANCE_CONNECTION_NAME={configured_instance}" in result.stdout @@ -1072,6 +1252,59 @@ def test_resolve_cloud_run_candidate_rejects_policy_write_selector_mismatch( ) in result.stderr +def test_resolve_cloud_run_candidate_verifies_database_identities(tmp_path): + gcloud_path, state_path = _fake_gcloud(tmp_path) + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + { + **_fake_gcloud_env(gcloud_path, state_path), + **_v2_target_env(), + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": (PRODUCTION_CLOUD_SQL_INSTANCE), + }, + ) + + assert result.returncode == 0, result.stderr + + +def test_resolve_cloud_run_candidate_rejects_database_identity_mismatch(tmp_path): + gcloud_path, state_path = _fake_gcloud(tmp_path) + state = json.loads(state_path.read_text(encoding="utf-8")) + state["candidate_env"]["V2_SUPABASE_PROJECT_REF"] = "z" * 20 + state_path.write_text(json.dumps(state), encoding="utf-8") + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + { + **_fake_gcloud_env(gcloud_path, state_path), + **_v2_target_env(), + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": (PRODUCTION_CLOUD_SQL_INSTANCE), + }, + ) + + assert result.returncode == 2 + assert "V2_SUPABASE_PROJECT_REF=zzzz" in result.stderr + + +def test_resolve_cloud_run_candidate_rejects_cloud_sql_attachment_mismatch(tmp_path): + gcloud_path, state_path = _fake_gcloud(tmp_path) + state = json.loads(state_path.read_text(encoding="utf-8")) + state["candidate_cloud_sql"] = "project:region:wrong-instance" + state_path.write_text(json.dumps(state), encoding="utf-8") + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + { + **_fake_gcloud_env(gcloud_path, state_path), + **_v2_target_env(), + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": (PRODUCTION_CLOUD_SQL_INSTANCE), + }, + ) + + assert result.returncode == 2 + assert "attaches Cloud SQL instance project:region:wrong-instance" in result.stderr + + def test_set_cloud_run_revision_promotes_and_rolls_back_exact_revisions(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) env = _fake_gcloud_env(gcloud_path, state_path) @@ -1404,6 +1637,7 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): "integration-tests-staging-cloud-run", ) cloud_run_promotion = _workflow_job_block(workflow, "promote-cloud-run-staging") + phase10_exercise = _workflow_job_block(workflow, "exercise-phase10-staging") production_gate = _workflow_job_block( workflow, "ensure-production-model-version-aligns-with-sim-api", @@ -1411,6 +1645,7 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): cloud_run_test_command = ( "python -m pytest tests/integration/test_cloud_run_candidate.py " "tests/integration/test_live_v2_metadata.py " + "tests/integration/test_live_v2_policies.py " "tests/integration/test_live_calculate.py " "tests/integration/test_live_economy.py " "tests/integration/test_live_budget_window_cache.py -v" @@ -1428,9 +1663,18 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): "API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}" in cloud_run_tests ) - assert "needs: promote-cloud-run-staging" in production_gate + assert "environment: staging" in cloud_run_tests + assert "V2_MIGRATION_DATABASE_URL" in cloud_run_tests + assert "needs: exercise-phase10-staging" in production_gate assert "- integration-tests-staging-cloud-run" not in production_gate assert "- integration-tests-staging-cloud-run" in cloud_run_promotion + assert "- promote-cloud-run-staging" in phase10_exercise + assert "DB_WRITE_POLICY: dual_write" in phase10_exercise + assert "run_phase10_staging_probe.sh activation" in phase10_exercise + assert "run_phase10_staging_probe.sh rollback" in phase10_exercise + assert "Restore exact Cloud SQL-only staging revision" in phase10_exercise + assert "actions/upload-artifact@v4" in phase10_exercise + assert "Fail after an incomplete Phase 10 staging exercise" in phase10_exercise assert "qualify-stage6-read-routes-staging" not in workflow assert "qualify_stage6_read_routes.sh" not in workflow assert "bash .github/scripts/set_cloud_run_revision.sh" in cloud_run_promotion diff --git a/tests/unit/v2/test_metadata_deployment.py b/tests/unit/v2/test_metadata_deployment.py index dedb61df9..03c5c025c 100644 --- a/tests/unit/v2/test_metadata_deployment.py +++ b/tests/unit/v2/test_metadata_deployment.py @@ -69,21 +69,29 @@ def test_schema_upgrade_precedes_atomic_catalog_publication() -> None: assert upgrade < current < drift +def test_policy_data_qualification_precedes_schema_upgrade() -> None: + workflow = _read(".github/workflows/seed-v2-database.yml") + + isolation = workflow.index("validate_database_environment.sh supabase") + qualification = workflow.index("scripts/qualify_v2_policy_migration.py") + upgrade = workflow.index("migrate_v2_metadata_schema.sh") + + assert isolation < qualification < upgrade + + def test_seeding_success_is_required_before_candidate_creation() -> None: workflow = _read(".github/workflows/push.yml") staging_seed = _job(workflow, "seed-v2-staging-database") production_seed = _job(workflow, "seed-v2-production-database") assert "deployment_environment: staging" in staging_seed - assert "migrate-v1-cloud-sql" in staging_seed + assert "migrate-v1-staging-cloud-sql" in staging_seed assert "seed-v2-staging-database" in _job( workflow, "deploy-cloud-run-staging", ) - assert ( - "needs: ensure-production-model-version-aligns-with-sim-api" in production_seed - ) + assert "migrate-v1-production-cloud-sql" in production_seed assert "deployment_environment: production" in production_seed assert "needs: seed-v2-production-database" in _job( workflow, diff --git a/tests/unit/v2/test_policy_migration_qualification.py b/tests/unit/v2/test_policy_migration_qualification.py index 170953b28..cc3f63201 100644 --- a/tests/unit/v2/test_policy_migration_qualification.py +++ b/tests/unit/v2/test_policy_migration_qualification.py @@ -73,6 +73,7 @@ def test_empty_target_is_qualified_in_a_rolled_back_read_only_transaction() -> N assert evidence.as_dict() == { "outcome": "ok", + "qualification": "performed", "environment": "staging", "project_ref": "abcdefghijklmnopqrst", "counts": { @@ -86,6 +87,48 @@ def test_empty_target_is_qualified_in_a_rolled_back_read_only_transaction() -> N assert engine.disposed +def test_applied_policy_revision_skips_predecessor_row_qualification() -> None: + connection = FakeConnection((1, 2, 3)) + engine = FakeEngine(connection) + + evidence = qualification.qualify_policy_migration_if_pending( + ENVIRONMENT, + engine_builder=lambda _settings: engine, + pending_checker=lambda _connection: False, + ) + + assert evidence.as_dict() == { + "outcome": "ok", + "qualification": "not-required", + "environment": "staging", + "project_ref": "abcdefghijklmnopqrst", + "counts": { + "policies": 0, + "policy_parameter_values": 0, + "user_policies": 0, + }, + } + assert connection.statements == ["SET TRANSACTION READ ONLY"] + assert connection.transaction.rolled_back + assert engine.disposed + + +def test_pending_policy_revision_runs_predecessor_row_qualification() -> None: + connection = FakeConnection((0, 0, 0)) + engine = FakeEngine(connection) + + evidence = qualification.qualify_policy_migration_if_pending( + ENVIRONMENT, + engine_builder=lambda _settings: engine, + pending_checker=lambda _connection: True, + ) + + assert evidence.required is True + assert connection.statements == ["SET TRANSACTION READ ONLY"] + assert connection.transaction.rolled_back + assert engine.disposed + + @pytest.mark.parametrize( "counts", [ @@ -122,7 +165,7 @@ def test_main_redacts_unexpected_errors( def fail() -> None: raise RuntimeError("postgresql://user:secret@private-host/database") - monkeypatch.setattr(qualification, "qualify_policy_migration_target", fail) + monkeypatch.setattr(qualification, "qualify_policy_migration_if_pending", fail) assert qualification.main() == 1 captured = capsys.readouterr() From 710bd3d901c1cde19e9ec612cd7919a98d4fafae Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:46:59 +0400 Subject: [PATCH 18/18] Fix Phase 10 staging failure probe --- .github/scripts/deploy_cloud_run_candidate.sh | 5 +++++ .github/scripts/run_phase10_staging_probe.sh | 4 +++- .github/workflows/push.yml | 2 +- tests/unit/test_cloud_run_deploy_scripts.py | 18 ++++++++++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index 8136894e7..84be00906 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -40,6 +40,11 @@ fi if [[ -n "${SIMULATION_ENTRYPOINT_URL:-}" ]]; then env_vars+=("SIMULATION_ENTRYPOINT_URL=${SIMULATION_ENTRYPOINT_URL}") fi +if [[ -n "${POLICYENGINE_API_STARTUP_WARMUP:-}" ]]; then + env_vars+=( + "POLICYENGINE_API_STARTUP_WARMUP=${POLICYENGINE_API_STARTUP_WARMUP}" + ) +fi secret_vars=( "POLICYENGINE_DB_PASSWORD=${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" diff --git a/.github/scripts/run_phase10_staging_probe.sh b/.github/scripts/run_phase10_staging_probe.sh index 00ab3b6ee..447cb901e 100644 --- a/.github/scripts/run_phase10_staging_probe.sh +++ b/.github/scripts/run_phase10_staging_probe.sh @@ -27,7 +27,9 @@ if [[ -z "${readonly_password}" ]]; then exit 1 fi -printf '::add-mask::%s\n' "${readonly_password}" +if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + printf '::add-mask::%s\n' "${readonly_password}" +fi POLICYENGINE_DB_READONLY_PASSWORD="${readonly_password}" \ python -m pytest \ "tests/integration/test_live_phase10_staging.py::${test_name}" \ diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 816af143a..e17b1c0f7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -474,7 +474,7 @@ jobs: CLOUD_RUN_IMAGE_URI: ${{ needs.deploy-cloud-run-staging.outputs.image }} CLOUD_RUN_TAG: ${{ steps.tags.outputs.failure }} CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com - CLOUD_RUN_STARTUP_PROBE: httpGet.path=/health-check,httpGet.port=8080,initialDelaySeconds=0,periodSeconds=10,failureThreshold=24,timeoutSeconds=5 + POLICYENGINE_API_STARTUP_WARMUP: "0" POLICYENGINE_DB_USER: ${{ vars.POLICYENGINE_DB_USER }} POLICYENGINE_DB_NAME: ${{ vars.POLICYENGINE_DB_NAME }} DB_WRITE_POLICY: dual_write diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index f16174f08..31e229c47 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -1003,6 +1003,21 @@ def test_deploy_cloud_run_candidate_dry_run_never_shifts_traffic(): assert result.stdout.count("DB_WRITE_POLICY=cloud_sql") == 1 +def test_deploy_cloud_run_candidate_passes_optional_startup_warmup_setting(): + result = _run_script( + ".github/scripts/deploy_cloud_run_candidate.sh", + _script_env( + **_required_runtime_env(), + CLOUD_RUN_IMAGE_URI="us-central1-docker.pkg.dev/project/repo/api:sha", + CLOUD_RUN_TAG="stage3-test", + POLICYENGINE_API_STARTUP_WARMUP="0", + ), + ) + + assert result.returncode == 0, result.stderr + assert "POLICYENGINE_API_STARTUP_WARMUP=0" in result.stdout + + def test_staging_and_production_use_distinct_cloud_run_runtime_identities(): workflow = _push_workflow() staging = _workflow_job_block(workflow, "deploy-cloud-run-staging") @@ -1670,6 +1685,9 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): assert "- integration-tests-staging-cloud-run" in cloud_run_promotion assert "- promote-cloud-run-staging" in phase10_exercise assert "DB_WRITE_POLICY: dual_write" in phase10_exercise + assert "Deploy controlled-failure staging revision" in phase10_exercise + assert 'POLICYENGINE_API_STARTUP_WARMUP: "0"' in phase10_exercise + assert "V2_FAILURE_DATABASE_URL_SECRET_RESOURCE" in phase10_exercise assert "run_phase10_staging_probe.sh activation" in phase10_exercise assert "run_phase10_staging_probe.sh rollback" in phase10_exercise assert "Restore exact Cloud SQL-only staging revision" in phase10_exercise