diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index eb1e5e7ab..ae1e49a44 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -3,6 +3,9 @@ Follow the repository's canonical engineering skills under `docs/engineering/skills/`. +For a new HTTP route or an existing route query-parameter contract change, +read `docs/engineering/skills/api-routes.md`. + For API v2 migration contract, route-group metadata, generated migration docs, or migration guard changes, read `docs/engineering/skills/migration_contracts.md`. @@ -13,5 +16,8 @@ reviewing test files. For SQLAlchemy model or Alembic migration work, read `docs/engineering/skills/alembic-migrations.md`. +For API v2 route, service, or database-access module additions or moves, read +`docs/engineering/skills/v2-code-organization.md`. + For pull requests, read `docs/engineering/skills/github-prs.md` before opening, replacing, or sharing a PR. diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index 37522c53d..c97672628 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -36,7 +36,7 @@ cloud_run_set_defaults() { # but adds no real scale-out delay because boot exceeds it either way. Sizing # rationale in docs/migration/cloud-run-operations.md. CLOUD_RUN_STARTUP_PROBE="${CLOUD_RUN_STARTUP_PROBE:-httpGet.path=/readiness-check,httpGet.port=${CLOUD_RUN_PORT},initialDelaySeconds=240,periodSeconds=10,failureThreshold=24,timeoutSeconds=5}" - CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}" + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}" CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}" diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index 87418163b..84be00906 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -20,6 +20,9 @@ env_vars=( "ROUTE_IMPL_HEALTH=${ROUTE_IMPL_HEALTH}" "ROUTE_IMPL_SPECIFICATION=${ROUTE_IMPL_SPECIFICATION}" "ROUTE_IMPL_METADATA=${ROUTE_IMPL_METADATA}" + "ROUTE_IMPL_POLICY=${ROUTE_IMPL_POLICY}" + "DB_READ_POLICY=${DB_READ_POLICY}" + "DB_WRITE_POLICY=${DB_WRITE_POLICY}" "SIM_COMPUTE_ECONOMY=old_gateway" "CLOUD_RUN_REVISION_TAG=${CLOUD_RUN_TAG}" "WEB_CONCURRENCY=${CLOUD_RUN_WEB_CONCURRENCY}" @@ -37,6 +40,11 @@ fi if [[ -n "${SIMULATION_ENTRYPOINT_URL:-}" ]]; then env_vars+=("SIMULATION_ENTRYPOINT_URL=${SIMULATION_ENTRYPOINT_URL}") fi +if [[ -n "${POLICYENGINE_API_STARTUP_WARMUP:-}" ]]; then + env_vars+=( + "POLICYENGINE_API_STARTUP_WARMUP=${POLICYENGINE_API_STARTUP_WARMUP}" + ) +fi secret_vars=( "POLICYENGINE_DB_PASSWORD=${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" @@ -63,7 +71,7 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \ --subnet "${CLOUD_RUN_VPC_SUBNET}" \ --vpc-egress "${CLOUD_RUN_VPC_EGRESS}" \ --service-account "${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \ - --add-cloudsql-instances "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" \ + --set-cloudsql-instances "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" \ --port "${CLOUD_RUN_PORT}" \ --cpu "${CLOUD_RUN_CPU}" \ --cpu-boost \ diff --git a/.github/scripts/migrate_v1_cloud_sql.sh b/.github/scripts/migrate_v1_cloud_sql.sh index 77d2bb23d..ae4928d0b 100644 --- a/.github/scripts/migrate_v1_cloud_sql.sh +++ b/.github/scripts/migrate_v1_cloud_sql.sh @@ -3,15 +3,19 @@ set -euo pipefail : "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME:?POLICYENGINE_DB_INSTANCE_CONNECTION_NAME is required}" +: "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET:?POLICYENGINE_DB_READONLY_PASSWORD_SECRET is required}" +: "${POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET:?POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET is required}" + +bash .github/scripts/validate_database_environment.sh cloud-sql readonly_password="$( gcloud secrets versions access latest \ - --secret policyengine-api-prod-db-readonly-password \ + --secret "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" \ --project policyengine-api )" migration_password="$( gcloud secrets versions access latest \ - --secret policyengine-api-prod-db-migration-password \ + --secret "${POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" \ --project policyengine-api )" diff --git a/.github/scripts/resolve_cloud_run_candidate_state.sh b/.github/scripts/resolve_cloud_run_candidate_state.sh index d5f9af4c0..b6e200084 100755 --- a/.github/scripts/resolve_cloud_run_candidate_state.sh +++ b/.github/scripts/resolve_cloud_run_candidate_state.sh @@ -78,26 +78,32 @@ image="$(jq -er ' | select(type == "string" and contains("@sha256:")) ' <<<"${revision_json}")" -route_selector_count=0 +deployment_selector_count=0 for selector in \ ROUTE_IMPL_HEALTH \ ROUTE_IMPL_SPECIFICATION \ - ROUTE_IMPL_METADATA; do + ROUTE_IMPL_METADATA \ + ROUTE_IMPL_POLICY \ + DB_READ_POLICY \ + DB_WRITE_POLICY; do if [[ -n "${!selector:-}" ]]; then - route_selector_count=$((route_selector_count + 1)) + deployment_selector_count=$((deployment_selector_count + 1)) fi done -if (( route_selector_count > 0 && route_selector_count < 3 )); then - echo "All Stage 6 route selectors are required when verifying candidate configuration" >&2 +if (( deployment_selector_count > 0 && deployment_selector_count < 6 )); then + echo "All route and policy database selectors are required when verifying candidate configuration" >&2 exit 2 fi -if (( route_selector_count == 3 )); then +if (( deployment_selector_count == 6 )); then for selector in \ ROUTE_IMPL_HEALTH \ ROUTE_IMPL_SPECIFICATION \ - ROUTE_IMPL_METADATA; do + ROUTE_IMPL_METADATA \ + ROUTE_IMPL_POLICY \ + DB_READ_POLICY \ + DB_WRITE_POLICY; do expected_value="${!selector}" actual_value="$(jq -r --arg name "${selector}" ' [ @@ -116,6 +122,57 @@ if (( route_selector_count == 3 )); then done fi +database_identity_count=0 +for setting in \ + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME \ + V2_SUPABASE_PROJECT_REF \ + V2_SUPABASE_ENVIRONMENT \ + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE; do + if [[ -n "${!setting:-}" ]]; then + database_identity_count=$((database_identity_count + 1)) + fi +done + +if (( database_identity_count > 0 && database_identity_count < 4 )); then + echo "All database identity settings are required when verifying candidate configuration" >&2 + exit 2 +fi + +if (( database_identity_count == 4 )); then + for setting in \ + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME \ + V2_SUPABASE_PROJECT_REF \ + V2_SUPABASE_ENVIRONMENT \ + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE; do + expected_value="${!setting}" + actual_value="$(jq -r --arg name "${setting}" ' + [ + .spec.containers[0].env[]? + | select(.name == $name) + | .value + ] + | if length == 1 then .[0] else "" end + ' <<<"${revision_json}")" + if [[ "${actual_value}" != "${expected_value}" ]]; then + printf 'Revision %s has %s=%s; expected %s\n' \ + "${revision}" "${setting}" "${actual_value:-}" \ + "${expected_value}" >&2 + exit 2 + fi + done + + attached_cloud_sql="$(jq -r ' + .metadata.annotations["run.googleapis.com/cloudsql-instances"] // empty + ' <<<"${revision_json}")" + if [[ "${attached_cloud_sql}" != \ + "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" ]]; then + printf 'Revision %s attaches Cloud SQL instance %s; expected %s\n' \ + "${revision}" "${attached_cloud_sql:-}" \ + "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" >&2 + exit 2 + fi +fi + if [[ -n "${CLOUD_RUN_EXPECTED_REVISION:-}" \ && "${revision}" != "${CLOUD_RUN_EXPECTED_REVISION}" ]]; then printf 'Candidate tag %s moved: expected revision %s, found %s\n' \ diff --git a/.github/scripts/run_phase10_staging_probe.sh b/.github/scripts/run_phase10_staging_probe.sh new file mode 100644 index 000000000..447cb901e --- /dev/null +++ b/.github/scripts/run_phase10_staging_probe.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET:?POLICYENGINE_DB_READONLY_PASSWORD_SECRET is required}" + +case "${1:-}" in + activation) + test_name="test_live_phase10_activation_failure_and_retry" + ;; + rollback) + test_name="test_live_phase10_cloud_sql_only_rollback" + ;; + *) + echo "Usage: $0 {activation|rollback}" >&2 + exit 1 + ;; +esac + +readonly_password="$( + gcloud secrets versions access latest \ + --secret "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" \ + --project policyengine-api +)" +if [[ -z "${readonly_password}" ]]; then + echo "The staging Cloud SQL read-only password must not be empty." >&2 + exit 1 +fi + +if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + printf '::add-mask::%s\n' "${readonly_password}" +fi +POLICYENGINE_DB_READONLY_PASSWORD="${readonly_password}" \ + python -m pytest \ + "tests/integration/test_live_phase10_staging.py::${test_name}" \ + -v diff --git a/.github/scripts/validate_cloud_run_deploy_env.sh b/.github/scripts/validate_cloud_run_deploy_env.sh index 03a2f2403..1ccd1e665 100755 --- a/.github/scripts/validate_cloud_run_deploy_env.sh +++ b/.github/scripts/validate_cloud_run_deploy_env.sh @@ -6,6 +6,8 @@ source .github/scripts/cloud_run_env.sh source .github/scripts/simulation_entrypoint_env.sh cloud_run_set_defaults +bash .github/scripts/validate_database_environment.sh runtime + # Cloud Run rejects deploys where the traffic tag and service name together # exceed 46 characters (they form the tag URL's DNS label). Fail fast here # with a clear message instead of at gcloud. @@ -41,6 +43,9 @@ cloud_run_require_env \ ROUTE_IMPL_HEALTH \ ROUTE_IMPL_SPECIFICATION \ ROUTE_IMPL_METADATA \ + ROUTE_IMPL_POLICY \ + DB_READ_POLICY \ + DB_WRITE_POLICY \ GATEWAY_AUTH_ISSUER \ GATEWAY_AUTH_AUDIENCE \ GATEWAY_AUTH_CLIENT_ID \ @@ -49,7 +54,8 @@ cloud_run_require_env \ for selector in \ ROUTE_IMPL_HEALTH \ ROUTE_IMPL_SPECIFICATION \ - ROUTE_IMPL_METADATA; do + ROUTE_IMPL_METADATA \ + ROUTE_IMPL_POLICY; do value="${!selector}" case "${value}" in flask_fallback|fastapi_native) ;; @@ -61,6 +67,21 @@ for selector in \ esac done +if [[ "${DB_READ_POLICY}" != "cloud_sql" ]]; then + printf '%s=%s is invalid; expected cloud_sql\n' \ + "DB_READ_POLICY" "${DB_READ_POLICY}" >&2 + exit 1 +fi + +case "${DB_WRITE_POLICY}" in + cloud_sql|dual_write) ;; + *) + printf '%s=%s is invalid; expected cloud_sql or dual_write\n' \ + "DB_WRITE_POLICY" "${DB_WRITE_POLICY}" >&2 + exit 1 + ;; +esac + selected_url_env="$( simulation_entrypoint_url_env_name "${SIM_ENTRYPOINT}" )" diff --git a/.github/scripts/validate_database_environment.sh b/.github/scripts/validate_database_environment.sh new file mode 100644 index 000000000..c619cd84b --- /dev/null +++ b/.github/scripts/validate_database_environment.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +set -euo pipefail + +mode="${1:-}" + +require_environment_variables() { + local missing=() + local name + + for name in "$@"; do + if [[ -z "${!name:-}" ]]; then + missing+=("${name}") + fi + done + + if (( ${#missing[@]} > 0 )); then + echo "Missing required database environment configuration: ${missing[*]}" >&2 + return 1 + fi +} + +require_environment_variables DEPLOYMENT_ENVIRONMENT + +case "${DEPLOYMENT_ENVIRONMENT}" in + staging|production) ;; + *) + printf 'DEPLOYMENT_ENVIRONMENT=%s is invalid; expected staging or production\n' \ + "${DEPLOYMENT_ENVIRONMENT}" >&2 + exit 1 + ;; +esac + +validate_cloud_sql() { + require_environment_variables \ + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME \ + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME + + if [[ "${DEPLOYMENT_ENVIRONMENT}" == "staging" ]]; then + if [[ "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" == \ + "${PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" ]]; then + echo "Staging Cloud SQL must use an instance distinct from production." >&2 + return 1 + fi + elif [[ "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" != \ + "${PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" ]]; then + echo "Production Cloud SQL does not match the configured production instance." >&2 + return 1 + fi + + local credential_setting_count=0 + local setting + for setting in \ + POLICYENGINE_DB_READONLY_PASSWORD_SECRET \ + POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET \ + PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET \ + PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET; do + if [[ -n "${!setting:-}" ]]; then + credential_setting_count=$((credential_setting_count + 1)) + fi + done + if (( credential_setting_count > 0 && credential_setting_count < 4 )); then + echo "All Cloud SQL migration credential resources are required together." >&2 + return 1 + fi + if (( credential_setting_count == 4 )); then + if [[ "${DEPLOYMENT_ENVIRONMENT}" == "staging" ]]; then + if [[ "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" == \ + "${PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" || \ + "${POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" == \ + "${PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" ]]; then + echo "Staging Cloud SQL migration credentials must be distinct from production." >&2 + return 1 + fi + elif [[ "${POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" != \ + "${PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET}" || \ + "${POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" != \ + "${PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET}" ]]; then + echo "Production Cloud SQL migration credentials do not match production resources." >&2 + return 1 + fi + fi +} + +validate_supabase() { + require_environment_variables \ + V2_SUPABASE_PROJECT_REF \ + V2_SUPABASE_ENVIRONMENT \ + PRODUCTION_V2_SUPABASE_PROJECT_REF + + if [[ "${DEPLOYMENT_ENVIRONMENT}" == "staging" ]]; then + if [[ "${V2_SUPABASE_ENVIRONMENT}" != "staging" ]]; then + echo "Staging V2_SUPABASE_ENVIRONMENT must be exactly staging." >&2 + return 1 + fi + if [[ "${V2_SUPABASE_PROJECT_REF}" == \ + "${PRODUCTION_V2_SUPABASE_PROJECT_REF}" ]]; then + echo "Staging Supabase must use a project distinct from production." >&2 + return 1 + fi + else + if [[ "${V2_SUPABASE_PROJECT_REF}" != \ + "${PRODUCTION_V2_SUPABASE_PROJECT_REF}" ]]; then + echo "Production Supabase does not match the configured production project." >&2 + return 1 + fi + if [[ "${V2_SUPABASE_ENVIRONMENT}" == "staging" ]]; then + echo "Production V2_SUPABASE_ENVIRONMENT must not be staging." >&2 + return 1 + fi + fi +} + +validate_runtime_secret() { + require_environment_variables \ + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \ + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \ + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE \ + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE + + if [[ "${DEPLOYMENT_ENVIRONMENT}" == "staging" ]]; then + if [[ "${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" == \ + "${PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" ]]; then + echo "Staging v1 runtime must use a database secret distinct from production." >&2 + return 1 + fi + if [[ "${V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" == \ + "${PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" ]]; then + echo "Staging v2 runtime must use a database secret distinct from production." >&2 + return 1 + fi + else + if [[ "${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" != \ + "${PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" ]]; then + echo "Production v1 runtime does not match the configured production secret." >&2 + return 1 + fi + if [[ "${V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" != \ + "${PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" ]]; then + echo "Production v2 runtime does not match the configured production secret." >&2 + return 1 + fi + fi +} + +case "${mode}" in + cloud-sql) + validate_cloud_sql + ;; + supabase) + validate_supabase + ;; + runtime) + validate_cloud_sql + validate_supabase + validate_runtime_secret + ;; + *) + echo "Usage: $0 {cloud-sql|supabase|runtime}" >&2 + exit 1 + ;; +esac diff --git a/.github/scripts/validate_phase10_staging_exercise_env.sh b/.github/scripts/validate_phase10_staging_exercise_env.sh new file mode 100644 index 000000000..f63522c95 --- /dev/null +++ b/.github/scripts/validate_phase10_staging_exercise_env.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euo pipefail + +required=( + DEPLOYMENT_ENVIRONMENT + ROUTE_IMPL_POLICY + DB_READ_POLICY + DB_WRITE_POLICY + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE + V2_FAILURE_DATABASE_URL_SECRET_RESOURCE + POLICYENGINE_DB_READONLY_PASSWORD_SECRET +) + +for setting in "${required[@]}"; do + if [[ -z "${!setting:-}" ]]; then + printf '%s is required for the Phase 10 staging exercise\n' "${setting}" >&2 + exit 1 + fi +done + +if [[ "${DEPLOYMENT_ENVIRONMENT}" != "staging" ]]; then + echo "The Phase 10 live exercise may run only against staging." >&2 + exit 1 +fi +if [[ "${ROUTE_IMPL_POLICY}" != "fastapi_native" ]]; then + echo "ROUTE_IMPL_POLICY must be fastapi_native for the Phase 10 exercise." >&2 + exit 1 +fi +if [[ "${DB_READ_POLICY}" != "cloud_sql" ]]; then + echo "DB_READ_POLICY must remain cloud_sql for the Phase 10 exercise." >&2 + exit 1 +fi +if [[ "${DB_WRITE_POLICY}" != "cloud_sql" ]]; then + echo "The staging candidate must begin with DB_WRITE_POLICY=cloud_sql." >&2 + exit 1 +fi +if [[ "${V2_FAILURE_DATABASE_URL_SECRET_RESOURCE}" == \ + "${V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}" ]]; then + echo "The controlled-failure database secret must differ from the valid staging secret." >&2 + exit 1 +fi +if [[ "${V2_FAILURE_DATABASE_URL_SECRET_RESOURCE}" == \ + "${PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE:-}" ]]; then + echo "The controlled-failure database secret must not identify production." >&2 + exit 1 +fi + +bash .github/scripts/validate_database_environment.sh runtime diff --git a/.github/workflows/migrate-v1-cloud-sql.yml b/.github/workflows/migrate-v1-cloud-sql.yml new file mode 100644 index 000000000..fbf22fe57 --- /dev/null +++ b/.github/workflows/migrate-v1-cloud-sql.yml @@ -0,0 +1,67 @@ +name: Migrate v1 Cloud SQL + +on: + workflow_call: + inputs: + deployment_environment: + description: Logical database environment + required: true + type: string + github_environment: + description: GitHub environment containing the Cloud SQL target + required: true + type: string + workflow_dispatch: + inputs: + deployment_environment: + description: Logical database environment + required: true + type: choice + options: + - staging + - production + github_environment: + description: GitHub environment containing the Cloud SQL target + required: true + type: environment + +jobs: + migrate: + name: Upgrade and verify v1 Cloud SQL + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: ${{ inputs.github_environment }} + permissions: + contents: read + id-token: write + env: + DEPLOYMENT_ENVIRONMENT: ${{ inputs.deployment_environment }} + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + POLICYENGINE_DB_READONLY_PASSWORD_SECRET: ${{ vars.POLICYENGINE_DB_READONLY_PASSWORD_SECRET }} + POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET: ${{ vars.POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET }} + PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET: ${{ vars.PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET }} + PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET: ${{ vars.PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Authenticate to GCP + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }} + - name: Set up GCloud + uses: google-github-actions/setup-gcloud@v2 + - name: Install dependencies + run: make install + - name: Start Cloud SQL Auth Proxy + run: bash .github/scripts/start_cloud_sql_proxy.sh + - name: Upgrade and verify v1 database + run: bash .github/scripts/migrate_v1_cloud_sql.sh + - name: Stop Cloud SQL Auth Proxy + if: always() + run: bash .github/scripts/stop_cloud_sql_proxy.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ffc5d0e0b..660617471 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -38,6 +38,22 @@ jobs: run: pip install ruff>=0.9.0 - name: Format check with ruff run: ruff format --check . + typecheck-v2: + name: Type-check API v2 + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup uv + uses: astral-sh/setup-uv@v6 + - name: Install locked dependencies + run: uv sync --frozen --extra dev + - name: Type-check API v2 + run: uv run --frozen --extra dev mypy quality-guards: name: Quality guards runs-on: ubuntu-latest diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 060780dcf..e17b1c0f7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -126,50 +126,25 @@ jobs: - name: Publish Git Tag run: ".github/publish-git-tag.sh" - migrate-v1-cloud-sql: - name: Upgrade v1 Cloud SQL schema - runs-on: ubuntu-latest + migrate-v1-staging-cloud-sql: + name: Upgrade staging v1 Cloud SQL schema needs: - ensure-staging-model-version-aligns-with-sim-api - publish-git-tag if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') - environment: production-database - permissions: - contents: read - id-token: write - env: - POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Authenticate to GCP - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }} - - name: Set up GCloud - uses: google-github-actions/setup-gcloud@v2 - - name: Install dependencies - run: make install - - name: Start Cloud SQL Auth Proxy - run: bash .github/scripts/start_cloud_sql_proxy.sh - - name: Upgrade and verify v1 database - run: bash .github/scripts/migrate_v1_cloud_sql.sh - - name: Stop Cloud SQL Auth Proxy - if: always() - run: bash .github/scripts/stop_cloud_sql_proxy.sh + uses: ./.github/workflows/migrate-v1-cloud-sql.yml + with: + deployment_environment: staging + github_environment: staging-database + secrets: inherit seed-v2-staging-database: name: Seed staging v2 database needs: - publish-git-tag - - migrate-v1-cloud-sql + - migrate-v1-staging-cloud-sql if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -184,26 +159,35 @@ jobs: needs: - ensure-staging-model-version-aligns-with-sim-api - publish-git-tag - - migrate-v1-cloud-sql + - migrate-v1-staging-cloud-sql - seed-v2-staging-database if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') environment: staging env: + DEPLOYMENT_ENVIRONMENT: staging SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} + ROUTE_IMPL_POLICY: ${{ vars.ROUTE_IMPL_POLICY }} + DB_READ_POLICY: ${{ vars.DB_READ_POLICY }} + DB_WRITE_POLICY: ${{ vars.DB_WRITE_POLICY }} CLOUD_RUN_SERVICE: policyengine-api-staging CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: staging CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-staging-runtime-cache-url:latest CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-staging-runtime-cache-ca:latest + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + PRODUCTION_V2_SUPABASE_PROJECT_REF: ${{ vars.PRODUCTION_V2_SUPABASE_PROJECT_REF }} V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} # Staging stays scale-to-zero, single instance: it exists for per-push # validation, not capacity. Both the revision-level (--min-instances) and # service-level (--min) floors are 0. @@ -235,9 +219,9 @@ jobs: run: make install - name: Run release tests run: >- - env -u ROUTE_IMPL_HEALTH -u ROUTE_IMPL_SPECIFICATION -u ROUTE_IMPL_METADATA - -u CLOUD_RUN_SERVICE -u CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT -u CLOUD_RUN_RUNTIME_CACHE_URL_SECRET -u CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET -u CLOUD_RUN_MIN_INSTANCES -u CLOUD_RUN_SERVICE_MIN_INSTANCES -u CLOUD_RUN_MAX_INSTANCES - -u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE make test + env -u DEPLOYMENT_ENVIRONMENT -u ROUTE_IMPL_HEALTH -u ROUTE_IMPL_SPECIFICATION -u ROUTE_IMPL_METADATA -u ROUTE_IMPL_POLICY -u DB_READ_POLICY -u DB_WRITE_POLICY + -u CLOUD_RUN_SERVICE -u CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT -u CLOUD_RUN_RUNTIME_CACHE_URL_SECRET -u CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET -u CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET -u CLOUD_RUN_MIN_INSTANCES -u CLOUD_RUN_SERVICE_MIN_INSTANCES -u CLOUD_RUN_MAX_INSTANCES + -u V2_SUPABASE_PROJECT_REF -u V2_SUPABASE_ENVIRONMENT -u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE -u PRODUCTION_V2_SUPABASE_PROJECT_REF -u PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE -u PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME make test env: RUNTIME_CACHE_MODE: local RUNTIME_CACHE_URL: redis://127.0.0.1:6379/0 @@ -301,6 +285,7 @@ jobs: if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') + environment: staging steps: - name: Checkout repo uses: actions/checkout@v4 @@ -309,12 +294,13 @@ jobs: with: python-version: "3.12" - name: Install staging test dependencies - run: pip install pytest httpx + run: pip install pytest httpx sqlalchemy psycopg[binary] - name: Run staging smoke test - run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v + run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v env: API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }} + V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} promote-cloud-run-staging: name: Promote staging Cloud Run traffic @@ -328,9 +314,16 @@ jobs: environment: staging env: CLOUD_RUN_SERVICE: policyengine-api-staging + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} + V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} + ROUTE_IMPL_POLICY: ${{ vars.ROUTE_IMPL_POLICY }} + DB_READ_POLICY: ${{ vars.DB_READ_POLICY }} + DB_WRITE_POLICY: ${{ vars.DB_WRITE_POLICY }} permissions: contents: read id-token: write @@ -376,10 +369,216 @@ jobs: if: ${{ always() && (steps.promote.outcome == 'failure' || steps.stable_health.outcome == 'failure') }} run: exit 1 + exercise-phase10-staging: + name: Exercise Phase 10 activation, failure, retry, and rollback + runs-on: ubuntu-latest + timeout-minutes: 45 + needs: + - deploy-cloud-run-staging + - promote-cloud-run-staging + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + environment: staging + env: + DEPLOYMENT_ENVIRONMENT: staging + SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} + SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} + POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + POLICYENGINE_DB_READONLY_PASSWORD_SECRET: ${{ vars.POLICYENGINE_DB_READONLY_PASSWORD_SECRET }} + POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET: ${{ vars.POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET }} + PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET: ${{ vars.PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET }} + PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET: ${{ vars.PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET }} + ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} + ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} + ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} + ROUTE_IMPL_POLICY: ${{ vars.ROUTE_IMPL_POLICY }} + DB_READ_POLICY: ${{ vars.DB_READ_POLICY }} + DB_WRITE_POLICY: ${{ vars.DB_WRITE_POLICY }} + CLOUD_RUN_SERVICE: policyengine-api-staging + CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: staging + CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-staging-runtime-cache-url:latest + CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-staging-runtime-cache-ca:latest + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} + V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + PRODUCTION_V2_SUPABASE_PROJECT_REF: ${{ vars.PRODUCTION_V2_SUPABASE_PROJECT_REF }} + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} + V2_FAILURE_DATABASE_URL_SECRET_RESOURCE: ${{ vars.V2_FAILURE_DATABASE_URL_SECRET_RESOURCE }} + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} + CLOUD_RUN_MIN_INSTANCES: "0" + CLOUD_RUN_SERVICE_MIN_INSTANCES: "0" + CLOUD_RUN_MAX_INSTANCES: "1" + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install live exercise dependencies + run: pip install pytest httpx sqlalchemy psycopg[binary] pymysql + - name: Verify isolated Phase 10 exercise configuration + run: bash .github/scripts/validate_phase10_staging_exercise_env.sh + - name: Authenticate deployment identity + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }} + - name: Set up GCloud + uses: google-github-actions/setup-gcloud@v2 + - name: Install jq + run: sudo apt-get install -y jq + - name: Compute exercise revision tags + id: tags + run: | + echo "dual_write=p10-on-${GITHUB_RUN_NUMBER}-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + echo "failure=p10-f-${GITHUB_RUN_NUMBER}-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + - name: Deploy dual-write staging revision + run: bash .github/scripts/deploy_cloud_run_candidate.sh + env: + CLOUD_RUN_IMAGE_URI: ${{ needs.deploy-cloud-run-staging.outputs.image }} + CLOUD_RUN_TAG: ${{ steps.tags.outputs.dual_write }} + CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com + POLICYENGINE_DB_USER: ${{ vars.POLICYENGINE_DB_USER }} + POLICYENGINE_DB_NAME: ${{ vars.POLICYENGINE_DB_NAME }} + DB_WRITE_POLICY: dual_write + GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} + GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} + GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} + GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} + - name: Resolve exact dual-write staging revision + id: dual_write + run: bash .github/scripts/resolve_cloud_run_candidate_state.sh >> "$GITHUB_OUTPUT" + env: + CLOUD_RUN_TAG: ${{ steps.tags.outputs.dual_write }} + DB_WRITE_POLICY: dual_write + - name: Wait for dual-write revision health + run: bash .github/scripts/health_check.sh "${{ steps.dual_write.outputs.url }}/readiness-check" + - name: Promote dual-write staging revision + id: promote_dual_write + run: bash .github/scripts/set_cloud_run_revision.sh + env: + CLOUD_RUN_TARGET_REVISION: ${{ steps.dual_write.outputs.revision }} + CLOUD_RUN_EXPECTED_CURRENT_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.revision }} + - name: Wait for dual-write stable URL health + run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.stable_url }}/readiness-check" + - name: Deploy controlled-failure staging revision + run: bash .github/scripts/deploy_cloud_run_candidate.sh + env: + CLOUD_RUN_IMAGE_URI: ${{ needs.deploy-cloud-run-staging.outputs.image }} + CLOUD_RUN_TAG: ${{ steps.tags.outputs.failure }} + CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com + POLICYENGINE_API_STARTUP_WARMUP: "0" + POLICYENGINE_DB_USER: ${{ vars.POLICYENGINE_DB_USER }} + POLICYENGINE_DB_NAME: ${{ vars.POLICYENGINE_DB_NAME }} + DB_WRITE_POLICY: dual_write + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.V2_FAILURE_DATABASE_URL_SECRET_RESOURCE }} + GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} + GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} + GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} + GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} + - name: Resolve exact controlled-failure staging revision + id: failure + run: bash .github/scripts/resolve_cloud_run_candidate_state.sh >> "$GITHUB_OUTPUT" + env: + CLOUD_RUN_TAG: ${{ steps.tags.outputs.failure }} + DB_WRITE_POLICY: dual_write + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.V2_FAILURE_DATABASE_URL_SECRET_RESOURCE }} + - name: Authenticate database inspection identity + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }} + - name: Start staging Cloud SQL Auth Proxy + run: bash .github/scripts/start_cloud_sql_proxy.sh + - name: Run activation, failure, and retry checks + id: activation + continue-on-error: true + run: bash .github/scripts/run_phase10_staging_probe.sh activation + env: + API_BASE_URL: ${{ steps.dual_write.outputs.url }} + PHASE10_FAILURE_API_BASE_URL: ${{ steps.failure.outputs.url }} + PHASE10_STATE_PATH: ${{ runner.temp }}/phase10-staging-evidence.json + PHASE10_CLOUD_SQL_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.revision }} + PHASE10_DUAL_WRITE_REVISION: ${{ steps.dual_write.outputs.revision }} + PHASE10_FAILURE_REVISION: ${{ steps.failure.outputs.revision }} + RUN_PHASE10_STAGING_EXERCISE: "1" + STAGING_API_TEST_PROBE_ID: phase10-${{ github.run_id }}-${{ github.run_attempt }} + V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} + - name: Stop staging Cloud SQL Auth Proxy after activation + if: always() + run: bash .github/scripts/stop_cloud_sql_proxy.sh + - name: Reauthenticate deployment identity for rollback + if: always() && steps.promote_dual_write.outcome == 'success' + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }} + - name: Restore exact Cloud SQL-only staging revision + id: rollback + if: always() && steps.promote_dual_write.outcome == 'success' + continue-on-error: true + run: bash .github/scripts/set_cloud_run_revision.sh + env: + CLOUD_RUN_TARGET_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.revision }} + CLOUD_RUN_EXPECTED_CURRENT_REVISION: ${{ steps.dual_write.outputs.revision }} + - name: Wait for restored staging revision health + id: rollback_health + if: always() && steps.rollback.outcome == 'success' + continue-on-error: true + run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.stable_url }}/readiness-check" + - name: Authenticate database inspection identity after rollback + if: always() && steps.rollback_health.outcome == 'success' + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }} + - name: Restart staging Cloud SQL Auth Proxy + if: always() && steps.rollback_health.outcome == 'success' + run: bash .github/scripts/start_cloud_sql_proxy.sh + - name: Run Cloud SQL-only rollback checks + id: rollback_probe + if: always() && steps.activation.outcome == 'success' && steps.rollback_health.outcome == 'success' + continue-on-error: true + run: bash .github/scripts/run_phase10_staging_probe.sh rollback + env: + API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.stable_url }} + PHASE10_STATE_PATH: ${{ runner.temp }}/phase10-staging-evidence.json + PHASE10_CLOUD_SQL_REVISION: ${{ needs.deploy-cloud-run-staging.outputs.revision }} + RUN_PHASE10_STAGING_EXERCISE: "1" + STAGING_API_TEST_PROBE_ID: phase10-${{ github.run_id }}-${{ github.run_attempt }} + V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} + - name: Stop staging Cloud SQL Auth Proxy after rollback + if: always() + run: bash .github/scripts/stop_cloud_sql_proxy.sh + - name: Retain non-secret Phase 10 staging evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: phase10-staging-evidence-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/phase10-staging-evidence.json + if-no-files-found: error + retention-days: 90 + - name: Fail after an incomplete Phase 10 staging exercise + if: >- + ${{ always() && ( + steps.activation.outcome != 'success' + || steps.rollback.outcome != 'success' + || steps.rollback_health.outcome != 'success' + || steps.rollback_probe.outcome != 'success' + ) }} + run: exit 1 + ensure-production-model-version-aligns-with-sim-api: name: Ensure production model version aligns with simulation API runs-on: ubuntu-latest - needs: promote-cloud-run-staging + needs: exercise-phase10-staging if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -395,9 +594,23 @@ jobs: - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh + migrate-v1-production-cloud-sql: + name: Upgrade production v1 Cloud SQL schema + needs: ensure-production-model-version-aligns-with-sim-api + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + uses: ./.github/workflows/migrate-v1-cloud-sql.yml + with: + deployment_environment: production + github_environment: production-database + secrets: inherit + seed-v2-production-database: name: Seed production v2 database - needs: ensure-production-model-version-aligns-with-sim-api + needs: + - ensure-production-model-version-aligns-with-sim-api + - migrate-v1-production-cloud-sql if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -415,19 +628,28 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine API') environment: production env: + DEPLOYMENT_ENVIRONMENT: production SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} ROUTE_IMPL_HEALTH: ${{ vars.ROUTE_IMPL_HEALTH }} ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} + ROUTE_IMPL_POLICY: ${{ vars.ROUTE_IMPL_POLICY }} + DB_READ_POLICY: ${{ vars.DB_READ_POLICY }} + DB_WRITE_POLICY: ${{ vars.DB_WRITE_POLICY }} CLOUD_RUN_SERVICE: policyengine-api CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: production CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-prod-runtime-cache-url:latest CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-prod-runtime-cache-ca:latest + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + PRODUCTION_V2_SUPABASE_PROJECT_REF: ${{ vars.PRODUCTION_V2_SUPABASE_PROJECT_REF }} V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ vars.PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }} # Sized by the Stage 2 qualification and the PR 4 host cutover — rationale # and numbers in docs/migration/cloud-run-operations.md ("Runtime shape and # scaling"). Warm capacity is expressed service-level (--min); the diff --git a/.github/workflows/seed-v2-database.yml b/.github/workflows/seed-v2-database.yml index 52c3673e2..0b02da004 100644 --- a/.github/workflows/seed-v2-database.yml +++ b/.github/workflows/seed-v2-database.yml @@ -21,8 +21,10 @@ jobs: timeout-minutes: 30 environment: ${{ inputs.deployment_environment }} env: + DEPLOYMENT_ENVIRONMENT: ${{ inputs.deployment_environment }} V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} + PRODUCTION_V2_SUPABASE_PROJECT_REF: ${{ vars.PRODUCTION_V2_SUPABASE_PROJECT_REF }} steps: - name: Checkout repo uses: actions/checkout@v4 @@ -34,6 +36,12 @@ jobs: uses: astral-sh/setup-uv@v6 - name: Install locked dependencies run: uv sync --frozen + - name: Verify the database environment is isolated + run: bash .github/scripts/validate_database_environment.sh supabase + - name: Qualify the pending policy migration + run: uv run python scripts/qualify_v2_policy_migration.py + env: + V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} - name: Upgrade and verify the v2 schema run: bash .github/scripts/migrate_v2_metadata_schema.sh env: diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index b69399a7b..40e362171 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -57,6 +57,8 @@ jobs: RUN_V2_CATALOG_COMPATIBILITY: "1" - name: Test v2 metadata publication and resource routes run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py + - name: Test v2 policy persistence and immediate v1 mirroring + run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Qualify production-scale v2 metadata publication run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication_qualification.py env: diff --git a/AGENTS.md b/AGENTS.md index 3837d2f22..a56d5ae68 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,9 @@ Canonical AI-facing engineering skills live under `docs/engineering/skills/`. Use those files as the source of truth across Codex, Claude, Copilot, and other AI tools. +When adding an HTTP route or changing an existing route's query-parameter +contract, read `docs/engineering/skills/api-routes.md`. + When changing API v2 migration contracts, route-group migration metadata, PR cutover plans, generated migration docs, or migration guard scripts, read `docs/engineering/skills/migration_contracts.md`. @@ -19,6 +22,9 @@ When changing SQLAlchemy models, Alembic configuration, database schemas, or migration revisions, read `docs/engineering/skills/alembic-migrations.md`. +When adding or moving API v2 route, service, or database-access modules, read +`docs/engineering/skills/v2-code-organization.md`. + ## GitHub PRs Read `docs/engineering/skills/github-prs.md` before opening, replacing, or diff --git a/CLAUDE.md b/CLAUDE.md index 6627f4b40..74b78e9f8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,6 +13,9 @@ rules here. ## Required Skill Lookup +Before adding an HTTP route or changing an existing route's query-parameter +contract, read `docs/engineering/skills/api-routes.md`. + Before opening, replacing, or sharing a PR, read `docs/engineering/skills/github-prs.md`. @@ -27,6 +30,9 @@ When changing SQLAlchemy models, Alembic configuration, database schemas, or migration revisions, read `docs/engineering/skills/alembic-migrations.md`. +When adding or moving API v2 route, service, or database-access modules, read +`docs/engineering/skills/v2-code-organization.md`. + ## Safety Boundaries Do not claim a route, database table, compute path, or deployment surface has diff --git a/Makefile b/Makefile index 41d451e53..8a97c7bb6 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,9 @@ test: quality-guards: python scripts/run_quality_guards.py +typecheck-v2: + uv run --frozen --extra dev mypy + debug-test: MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 pytest -vv --durations=0 tests diff --git a/changelog.d/stage-10.added.md b/changelog.d/stage-10.added.md new file mode 100644 index 000000000..35cdf955f --- /dev/null +++ b/changelog.d/stage-10.added.md @@ -0,0 +1 @@ +Add US and UK API v2 policies and user-policy associations with immediate API v1 mutation mirroring while other v1-supported countries remain Cloud SQL-only. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index 8cab684a5..92691c1c1 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -7,8 +7,8 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | Metric | Count | | --- | ---: | | route group count | 9 | -| workflow count | 8 | -| request count | 32 | +| workflow count | 11 | +| request count | 43 | | db entity count | 6 | | sim flow count | 3 | @@ -19,7 +19,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check` | `none` | `none` | | `specification` | `specification` | `none` | `none` | | `metadata` | `metadata`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | -| `policy` | `policy`, `policies`, `user-policy` | `policy` | `none` | +| `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | | `household` | `household`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | | `simulation` | `simulation`, `simulations` | `simulation` | `economy` | @@ -39,6 +39,41 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/us/policy/{policy_id}` | 200 | `policy` | `status`, `message`, `result` | | `GET` | `/us/policies` | 200 | `policy` | `result` | +### `policy_resources_v2` + +- Current contract: `typed_v2_resources` +- Future owner: PR 10: Policy Migration + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/v2/policies?country_id=us` | 201 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at` | +| `GET` | `/v2/policies/{policy_id}?country_id=us` | 200 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at` | +| `GET` | `/v2/policies?country_id=us` | 200 | `policy` | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | + +### `saved_policy_v1_compatibility` + +- Current contract: `api_v1_compatible` +- Future owner: PR 10: Policy Migration + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/us/user-policy` | 201 | `policy` | `status`, `message`, `result.id`, `result.reform_id`, `result.reform_label`, `result.baseline_id`, `result.user_id` | +| `GET` | `/us/user-policy/{user_id}` | 200 | `policy` | `status`, `message`, `result` | +| `PUT` | `/us/user-policy` | 200 | `policy` | `status`, `message`, `result.id` | + +### `user_policy_associations_v2` + +- Current contract: `typed_v2_resources` +- Future owner: PR 10: Policy Migration + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/v2/user-policies?country_id=us` | 201 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.user_id`, `result.item.policy_id`, `result.item.name`, `result.item.description`, `result.item.created_at`, `result.item.updated_at` | +| `GET` | `/v2/user-policies/{association_id}?country_id=us` | 200 | `policy` | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.user_id`, `result.item.policy_id`, `result.item.name`, `result.item.description`, `result.item.created_at`, `result.item.updated_at` | +| `GET` | `/v2/user-policies?country_id=us&user_id={user_id}` | 200 | `policy` | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | +| `PATCH` | `/v2/user-policies/{association_id}?country_id=us` | 200 | `policy` | `status`, `message`, `result.item.id`, `result.item.name`, `result.item.description`, `result.item.updated_at` | +| `DELETE` | `/v2/user-policies/{association_id}?country_id=us` | 204 | `policy` | | + ### `household_save_edit_read` - Current contract: `api_v1_compatible` diff --git a/docs/engineering/skills/README.md b/docs/engineering/skills/README.md index a3248f5b2..8d641a20d 100644 --- a/docs/engineering/skills/README.md +++ b/docs/engineering/skills/README.md @@ -9,6 +9,8 @@ first, then keep adapters thin. Current skills: +- `api-routes.md`: mandatory shared typed query-parameter conventions for new + routes and existing query-contract changes. - `alembic-migrations.md`: mandatory autogenerated Alembic revision workflow, deployment safeguards, and migration validation. - `github-prs.md`: PR workflow and migration PR handoff expectations. @@ -16,3 +18,5 @@ Current skills: metadata, generated migration artifacts, and quality guards. - `testing.md`: focused test commands and dependency boundaries for migration work. +- `v2-code-organization.md`: mandatory API v2 route, service, and database-access + package boundaries and literal module-naming rules. diff --git a/docs/engineering/skills/api-routes.md b/docs/engineering/skills/api-routes.md new file mode 100644 index 000000000..1b6a4b91a --- /dev/null +++ b/docs/engineering/skills/api-routes.md @@ -0,0 +1,120 @@ +# API Route Query Contracts + +Read this guidance before adding an HTTP route or changing the query-parameter +contract of an existing route. + +## Scope + +Every new route that accepts query parameters must define them through the +repository's shared typed query-parameter mechanism. When a change adds, +removes, renames, or changes the meaning of a query parameter on an existing +route, migrate that route's complete query contract to the shared mechanism as +part of the same change. + +Do not refactor an unrelated existing route merely because another part of its +module changes. This rule applies when the route is new or its public query +contract changes. + +## Canonical Parameter Meanings + +The framework-neutral source is `policyengine_api/query_parameters.py`. +Resource query models compose `CountryQuery`, `CatalogQuery`, +`PaginationQuery`, and the canonical annotated field types defined there. +FastAPI routes obtain dependencies through +`policyengine_api/fastapi_routes/query_parameters.py::query_dependency`. +Flask routes use `parse_multidict_query` when a reviewed legacy query contract +is migrated. Route modules must not reproduce these adapters. + +Reuse the shared definition whenever a query field has an established meaning. +The definition owns all of the following behavior: + +- public parameter name; +- Python and OpenAPI types; +- normalization and coercion; +- required or optional status; +- default value; +- length, numeric, enumeration, and collection bounds; +- scalar or list multiplicity; +- validation error semantics. + +Country, PolicyEngine.py version, pagination, UUID resource filters, and other +repeated filter concepts must not be redefined independently in route modules. +A resource-specific query schema should compose canonical fields and add only +filters whose meaning is specific to that resource. + +`country_id` is required only when the resource contract is country-scoped. Do +not add it to a route that has no country-dependent behavior merely for visual +uniformity. + +## Parsing Rules + +- Reject unknown query parameters rather than ignoring misspellings. +- Reject a scalar query parameter supplied more than once rather than selecting + an arbitrary value. +- Accept repeated keys only when the canonical field is explicitly list-valued. +- Keep query, path, and request-body fields in their documented locations; do + not move an identifier between them to reuse a schema. +- Do not maintain a second manual parser with different defaults or coercion. +- Do not copy parsing behavior from a legacy route when it conflicts with the + canonical typed definition. + +FastAPI routes should consume composed query schemas as typed dependencies so +the runtime validation and generated OpenAPI schema have the same source. A +Flask route whose query contract changes should use a thin adapter from +`request.args` into the same canonical schema rather than calling `get`, +`json.loads`, `int`, or similar coercion independently for each field. + +The Phase 10 policy schemas demonstrate the required composition: + +- `PolicyCreateQuery` for policy creation; +- `PolicyDetailQuery` for country-scoped policy detail; +- `PolicyCollectionQuery` for exact model filtering and pagination; +- `CountryQuery` for association create, detail, update, and delete; +- `UserPolicyCollectionQuery` for association user/policy filtering and + pagination. + +## Compatibility and Documentation + +Changing a query parameter's name, type, default, bounds, multiplicity, or +normalization changes the public route contract. Preserve existing behavior +unless the change explicitly authorizes a contract revision. For API v2 +migration routes, also read `migration_contracts.md` and update the migration +registry, workflow contracts, generated documentation, and stable OpenAPI +fields when applicable. + +The route's OpenAPI operation must expose every accepted query field with the +same required status, type, default, and bounds enforced at runtime. Do not +document query parameters accepted only by an untyped fallback parser. + +## CORS Preflight Requests + +The outer ASGI application owns cross-origin request handling through +Starlette's `CORSMiddleware`. Do not add resource-specific `OPTIONS` operations +or CORS headers to FastAPI route functions. A browser preflight contains +`Origin` and `Access-Control-Request-Method`; the middleware must answer it +before resource routing, including when the eventual resource method is +`POST`, `PATCH`, or `DELETE`. An ordinary `OPTIONS` request without those +headers continues through normal route resolution. + +When a new public HTTP method or request header is added, verify that the +application-level CORS configuration permits it. Tests must cover the +preflight response and an error response, and browser-readable response +headers must be included in `Access-Control-Expose-Headers` when applicable. + +## Required Verification + +For each new or changed query contract, cover the applicable cases: + +- required parameters are absent; +- optional parameters use their canonical defaults; +- valid values receive canonical normalization; +- invalid types and out-of-range values are rejected; +- unknown parameters are rejected; +- duplicate scalar parameters are rejected; +- explicitly list-valued parameters accept the documented repeated form; +- OpenAPI declares the runtime name, type, required status, default, and bounds; +- parameters reused by multiple routes behave identically. + +Use `docs/engineering/skills/testing.md` to select the appropriate test layer +and commands. Route behavior that changes a migration contract also requires +the focused migration checks documented in `migration_contracts.md`. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 8af342cde..fa8b0d573 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -104,11 +104,14 @@ Redis server; missing managed-cache configuration fails closed; Cloud SQL and existing routes/compute remain primary; and generated migration-contract artifacts remain current. -Cloud Run must receive `ROUTE_IMPL_HEALTH`, `ROUTE_IMPL_SPECIFICATION`, and -`ROUTE_IMPL_METADATA` from the selected GitHub environment. Candidate -resolution must verify those values on the exact revision. Staging promotion -must wait for the complete Cloud Run staging integration suite against the -tagged candidate. +Cloud Run must receive `ROUTE_IMPL_HEALTH`, `ROUTE_IMPL_SPECIFICATION`, +`ROUTE_IMPL_METADATA`, `ROUTE_IMPL_POLICY`, `DB_READ_POLICY`, and +`DB_WRITE_POLICY` from the selected GitHub environment. Candidate resolution +must verify all six values on the exact revision. Staging promotion must wait +for the complete Cloud Run staging integration suite against the tagged +candidate. ASGI route tests must also prove that standard CORS middleware +answers browser preflight requests before route resolution and adds CORS +headers to error responses. For PR 3 Cloud Run candidate deployment changes, verify the command-building guards, workflow track structure, ASGI compatibility, and container build: @@ -155,3 +158,73 @@ ruff check Commit only after formatting succeeds and changed Python files pass lint. If a broader repo-wide lint command fails on unrelated pre-existing issues, include that result in the handoff instead of hiding it. + +## Phase 10 Policy Migration + +Run the configured static type check for the Phase 10 v2 query, route, +application-service, metadata-read, policy-create and read, and association +CRUD modules. The configured file set deliberately excludes the existing v1 +implementation: + +```bash +uv run --frozen --extra dev mypy +``` + +Run the shared query, SQLModel, native policy and association, v1 compatibility, +configuration, readiness, and observability tests together: + +```bash +uv run pytest \ + tests/unit/test_query_parameters.py \ + tests/unit/v2/test_models.py \ + tests/unit/v2/test_model_persistence.py \ + tests/unit/v2/test_data_crud_boundaries.py \ + tests/unit/v2/test_policy_routes.py \ + tests/unit/v2/test_user_policy_routes.py \ + tests/unit/v2/test_user_policy_service.py \ + tests/unit/services/test_policy_service.py \ + tests/unit/services/test_user_policy_service.py \ + tests/unit/services/test_policy_mirroring.py \ + tests/unit/services/test_user_policy_mirroring.py \ + tests/unit/routes/test_policy_dual_write_routes.py \ + tests/unit/routes/test_user_policy_dual_write_routes.py \ + tests/unit/test_migration_flags.py \ + tests/unit/test_readiness.py \ + tests/contract -q +``` + +Use only the reviewed disposable PostgreSQL target for the v2 lifecycle, +persistence, and cross-database transaction tests: + +```bash +V2_ALEMBIC_DISPOSABLE_TEST=1 \ +V2_MIGRATION_DATABASE_URL="postgresql+psycopg://.../policyengine_v2_alembic_test" \ +uv run pytest \ + tests/integration/test_alembic_v2_lifecycle.py \ + tests/integration/test_v2_policy_persistence.py \ + tests/integration/test_v1_policy_dual_write.py \ + tests/integration/test_v2_user_policy_mirroring.py \ + tests/integration/test_v1_user_policy_dual_write.py -q +``` + +Continue to run the isolated v1 MySQL lifecycle and compatibility suite because +Phase 10 adds source revision and mirror-event storage to Cloud SQL while it +must preserve every v1 read and response contract: + +```bash +uv run pytest \ + tests/integration/test_alembic_mysql_lifecycle.py \ + tests/contract/test_v1_route_contracts.py \ + tests/unit/services/test_policy_service.py \ + tests/unit/services/test_user_policy_service.py \ + tests/unit/routes/test_policy_dual_write_routes.py \ + tests/unit/routes/test_user_policy_dual_write_routes.py -q +``` + +Finally regenerate and validate migration contracts: + +```bash +python scripts/export_migration_contracts.py +python scripts/run_quality_guards.py +uv run pytest tests/contract tests/unit/test_migration_contract_artifacts.py -q +``` diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md new file mode 100644 index 000000000..569e1cc41 --- /dev/null +++ b/docs/engineering/skills/v2-code-organization.md @@ -0,0 +1,154 @@ +# API v2 Code Organization + +Organize API v2 code first by resource and then by one explicit technical +responsibility. Route modules must not open database sessions or construct SQL. +Service modules must sequence work but must not construct or execute SQL. + +## HTTP adapters + +Resource-specific FastAPI code lives under `policyengine_api/fastapi_routes/v2/`: + +```text +errors.py +policies/ + request_models.py + response_models.py + routes.py +user_policies/ + request_models.py + response_models.py + routes.py +metadata/ + response_models.py + common.py + *_routes.py +``` + +`errors.py` defines the strict error envelope and serialization function shared +by every API v2 resource. Application-wide exception handling distinguishes +API v2 from non-v2 requests but must not select error behavior by matching +individual resource paths. + +Request models describe HTTP bodies. Response models describe public response +envelopes and OpenAPI output. Route functions handle HTTP-only conditions, +invoke one service method, and convert typed failures to HTTP responses. + +## Resource service packages + +New and moved API v2 resource code uses this layout: + +```text +services/v2// + services.py + validators.py + transformations.py + types.py + database_session.py + database_connectors/ + __init__.py + creates.py + reads.py + updates.py + deletes.py +``` + +Only add CRUD connector modules for operations the resource supports. Append a +specific descriptor when one operation file would become too broad, such as +`reads_variables.py` or `reads_parameter_tree.py`. + +Policies currently have `creates.py` and `reads.py` because policies are +immutable. Metadata currently has only read connector modules because metadata +routes are read-only. User-policy associations have `creates.py`, `reads.py`, +`updates.py`, and `deletes.py` because they support all four operations. + +### `services.py` + +Define the overarching functions and classes called by route functions or +other application services. A service determines operation order, calls pure +validation and transformation functions, and passes a database session to one +or more connector functions. A service may expose an entrypoint that accepts an +existing session when several resources must change atomically in one caller- +owned transaction. + +Do not construct SQL expressions, call `Session.exec`, or define HTTP response +models in this module. + +### `validators.py` + +Define functions that inspect already-available values and either return a +validated value or raise a typed exception. Validators must not load records, +open sessions, execute SQL, mutate database rows, or construct HTTP responses. + +When validation depends on stored state, a database connector loads the +required rows and the service passes those rows to a validator. For example, +policy catalog membership is checked only after a read connector returns the +selected catalog and matching parameter identifiers. + +### `transformations.py` + +Define deterministic conversions between representations. Examples include +converting database rows to service result types, translating a detached v1 +snapshot into v2 input using already-loaded catalog records, and producing a +canonical byte representation for content deduplication. + +Transformation functions must not open sessions, execute SQL, mutate database +rows, own transaction behavior, or construct HTTP responses. + +### `types.py` + +Define framework-independent Pydantic models, dataclasses, enums, and type +aliases exchanged between routes, services, validators, transformations, and +database connectors. Names should describe the represented data, such as +`PolicyCreationInput`, not an architectural pattern such as “command.” + +Types may enforce their own field-level invariants through Pydantic validation, +but multi-record or catalog validation belongs in `validators.py`. + +### `database_session.py` + +Define the resource's database-session lifetime container. It may open and +close sessions, begin transactions, commit, or roll back. It must not construct +SQL, choose records, validate business rules, transform records into response +types, or determine multi-step operation order. + +Composition code should construct this container and inject it into the +service. A request-scoped read service may wrap one already-open session; a +write service may wrap a session factory and expose read and transaction +context managers. + +### `database_connectors/` + +Every function that constructs or executes SQL, calls `Session.get`, or mutates +ORM rows belongs in this package. Organize connector files by CRUD operation: + +- `creates.py` inserts rows or adds new ORM objects. +- `reads.py` contains `SELECT` operations and `Session.get` calls. +- `updates.py` changes existing rows. +- `deletes.py` removes rows. + +Connector functions receive a session from the service. They do not open, +commit, roll back, or close it. They do not call route functions, perform +request-level or business validation, or convert rows into public response +types. Connector modules do not call one another to sequence a workflow; the +service makes that ordering explicit. + +Do not use `repository` as a generic synonym for SQL access. Reserve that term +for an intentional Repository-pattern abstraction with a stable interface that +hides interchangeable persistence implementations. API v2 currently uses +direct database connector functions. + +## Dependency direction + +The normal dependency direction is: + +```text +route -> service -> database session + database connectors -> SQLModel tables + -> validators + -> transformations + -> types +``` + +Database connectors may consume service-layer input types when inserting or +updating rows. They must not import route modules. Validators and +transformations may consume types and already-loaded database model instances, +but must not depend on SQLAlchemy or SQLModel query/session APIs. diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index fa25f245c..bf976fccb 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,10 +1,10 @@ { "metadata": { "db_entity_count": 6, - "request_count": 32, + "request_count": 43, "route_group_count": 9, "sim_flow_count": 3, - "workflow_count": 8 + "workflow_count": 11 }, "route_groups": [ { @@ -48,7 +48,8 @@ "path_segments": [ "policy", "policies", - "user-policy" + "user-policy", + "user-policies" ], "sim_flow": null }, @@ -136,6 +137,181 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "PR 10: Policy Migration", + "name": "policy_resources_v2", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/v2/policies?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/v2/policies/{policy_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/v2/policies?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.items", + "result.offset", + "result.limit", + "result.has_more" + ] + } + ] + }, + { + "current_contract": "api_v1_compatible", + "future_owner_pr": "PR 10: Policy Migration", + "name": "saved_policy_v1_compatibility", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/us/user-policy", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.id", + "result.reform_id", + "result.reform_label", + "result.baseline_id", + "result.user_id" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/us/user-policy/{user_id}", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result" + ] + }, + { + "expected_status": 200, + "method": "PUT", + "path": "/us/user-policy", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.id" + ] + } + ] + }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "PR 10: Policy Migration", + "name": "user_policy_associations_v2", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/v2/user-policies?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.user_id", + "result.item.policy_id", + "result.item.name", + "result.item.description", + "result.item.created_at", + "result.item.updated_at" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/v2/user-policies/{association_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.user_id", + "result.item.policy_id", + "result.item.name", + "result.item.description", + "result.item.created_at", + "result.item.updated_at" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/v2/user-policies?country_id=us&user_id={user_id}", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.items", + "result.offset", + "result.limit", + "result.has_more" + ] + }, + { + "expected_status": 200, + "method": "PATCH", + "path": "/v2/user-policies/{association_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.name", + "result.item.description", + "result.item.updated_at" + ] + }, + { + "expected_status": 204, + "method": "DELETE", + "path": "/v2/user-policies/{association_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [] + } + ] + }, { "current_contract": "api_v1_compatible", "future_owner_pr": "PR 11: Household Migration", diff --git a/docs/migration/stage-10-v2-policies.md b/docs/migration/stage-10-v2-policies.md new file mode 100644 index 000000000..c9f33e541 --- /dev/null +++ b/docs/migration/stage-10-v2-policies.md @@ -0,0 +1,200 @@ +# Stage 10 V2 Policy Deployment + +This runbook applies the reviewed v2 policy schema, activates native v2 policy +resources, and optionally requires immediate v1-to-v2 mirroring. V1 reads and +integer identifiers remain in Cloud SQL throughout this stage. + +## Preconditions + +1. Stage 9 metadata catalogs must be initialized for the v2-supported US and UK + and the running PolicyEngine.py version. Policy creation does not fall back + to v1 metadata or another catalog version. +2. The runtime Supabase URL and target identity settings must identify the same + reviewed project and require TLS. Use either the direct endpoint or + Supavisor session mode on port 5432. The runtime rejects transaction-pooling + endpoints on port 6543 because they cannot apply the per-session statement + timeout. +3. Before applying the Phase 10 revisions beginning with `711ec2f0a5a5`, run the dormant-table qualification + against the exact migration target: + + ```bash + V2_MIGRATION_DATABASE_URL="postgresql+psycopg://..." \ + V2_SUPABASE_PROJECT_REF="reviewed-project-reference" \ + V2_SUPABASE_ENVIRONMENT="staging" \ + python scripts/qualify_v2_policy_migration.py + ``` + + Continue only when the result reports zero policies, policy-owned parameter + values, and user-policy associations requiring preservation. A nonzero count + requires a separate preservation decision. +4. Staging must use independently writable copies of both production databases. + The staging Supabase project reference, Cloud SQL instance connection name, + v1 password secret, and v2 runtime URL secret must all differ from their + production values. The staging Cloud Run runtime identity must not have + access to either production database secret. + +The repository-level `PRODUCTION_*` variables record non-secret production +identities for comparison. The `staging`, `staging-database`, `production`, and +`production-database` GitHub environments supply their own target identities +and credential-resource names. The deployment and migration scripts stop +before connecting when the selected environment is missing or a staging value +equals its production counterpart. + +## Schemas Before Traffic + +Apply both generated revisions before deploying code that can enable immediate +v1 saved-policy mirroring. The MySQL revision adds the source revision and +ordered event records; the PostgreSQL revision adds the destination's last +applied source revision. + +```bash +ALEMBIC_DATABASE_URL="mysql+pymysql://..." \ +uv run alembic -c alembic-v1.ini upgrade head +``` + +```bash +V2_MIGRATION_DATABASE_URL="postgresql+psycopg://..." \ +V2_SUPABASE_PROJECT_REF="reviewed-project-reference" \ +V2_SUPABASE_ENVIRONMENT="staging" \ +uv run alembic -c alembic-v2.ini upgrade head +``` + +Run each Alembic `check` command against the same corresponding target and +confirm no metadata/schema difference. The relevant generated revisions are +`3d6e8f553ca5` for MySQL and `af34023a728f` for the current PostgreSQL head. + +The release workflow applies and verifies both staging schemas before building +the staging candidate. It replaces, rather than appends to, the candidate's +Cloud SQL attachment and verifies the deployed revision's Cloud SQL attachment, +database identity environment variables, route implementation settings, and +database read/write settings. It then performs the complete activation, +controlled-failure, retry, and application-rollback exercise described below. +Production schema jobs are not eligible to run until that job has restored and +verified the exact Cloud SQL-only staging revision. + +## Activation + +The GitHub `staging` and `production` environments must define all three Phase +10 deployment variables: + +```text +ROUTE_IMPL_POLICY=flask_fallback +DB_READ_POLICY=cloud_sql +DB_WRITE_POLICY=cloud_sql +``` + +The deployment workflow passes these values to Cloud Run, rejects missing or +invalid values before deployment, and verifies the exact values on the tagged +candidate revision before testing or promotion. These initial values do not +activate native policy readiness or v1 mirroring. + +Native `/v2/policies` and `/v2/user-policies` routes use only the server-side +Supabase connection. The routes are registered as preview resources; +`ROUTE_IMPL_POLICY=fastapi_native` declares them operational for deployment +readiness validation without moving v1 routes away from Flask. + +Native user-policy requests use an existing `users.id` UUID. V1 saved-policy +mirroring keeps the opaque Cloud SQL user identifier out of that foreign-key +column: `legacy_user_mappings` maps the exact v1 string one-to-one to a v2 user +UUID. First use creates a minimal v2 user and mapping in the same Supabase +transaction as the association; later saves and retries reuse that UUID. +`first_name`, `last_name`, and `email` are null for these transition-created +users because the v1 saved-policy record does not supply them. This stage does +not add or infer Auth0 identifiers. + +Keep the initial v1 settings explicit: + +```text +DB_READ_POLICY=cloud_sql +DB_WRITE_POLICY=cloud_sql +``` + +After native lifecycle checks pass, require immediate mirroring for US and UK +v1 policy and saved-policy mutations: + +```text +DB_READ_POLICY=cloud_sql +DB_WRITE_POLICY=dual_write +``` + +The automated exercise deploys this selection as a distinct no-traffic +revision, verifies the exact immutable image and environment configuration, +and then assigns staging traffic to that revision. It also deploys a separate +no-traffic revision whose staging-only Secret Manager resource contains an +intentionally invalid password for the same staging Supabase project. That +revision uses `/health-check` only for process startup so the test can send a +real v1 write and verify the HTTP 503 response produced by an unavailable v2 +database. The invalid secret is accessible only to the staging runtime service +account and does not identify a production resource. + +Under this selection, a core-policy mutation commits Cloud SQL first and then +completes its policy transaction in Supabase. A saved-policy mutation commits +its source row, incremented revision, and complete event in one Cloud SQL +transaction. The same request then processes that source's pending events in +revision order and records `processed_at` only after the corresponding +Supabase transaction commits. + +Canada, Nigeria, and Israel remain supported by v1 but have no v2 catalog in +this phase. Their policy and saved-policy mutations continue using Cloud SQL +only under `DB_WRITE_POLICY=dual_write`: they create no v2 snapshot or mirror +event and do not access Supabase. + +A Supabase failure returns HTTP 503. An identical client retry reads the +already committed v1 row, appends the next revision when applicable, and first +replays any retained earlier event. Destination revision and fingerprint +checks make a replay safe if Supabase committed but the Cloud SQL processing +marker did not. There is no background processor or reconciliation process. + +The v2 runtime applies one five-second value to pool acquisition, PostgreSQL +connection establishment, and each SQL statement. The API does not retry these +operations internally. Native v2 routes and v1 mirroring return a secret-safe +HTTP 503 for a timeout or other SQLAlchemy database failure so the caller can +retry the complete request. + +Browser preflight requests are handled by the outer ASGI CORS middleware before +FastAPI or Flask route resolution. It permits the public HTTP methods and +request headers used by v1 and v2, exposes `X-PolicyEngine-Request-Id`, and +applies CORS headers to typed errors and service-unavailable responses. Route +functions do not implement separate preflight behavior. + +## Monitoring + +Monitor structured events with metric names `v1_policy_mirror_operations` and +`v1_user_policy_mirror_operations`. Alert on `outcome=error`, grouped by +`failure_category` and country. Events include the legacy integer ID, +destination UUID when committed, attempted and completed database sources, and +duration. They do not include policy JSON, presentation data, database URLs, or +credentials. + +Verify during staged activation that: + +- native policy and association create/read/list/update/delete operations use + the initialized catalog and Supabase only; +- both newly created and existing v1 policies receive durable mappings; +- v1 saved-policy label updates change association `name`, while v1-only field + updates change only the mapping fingerprint; +- saved-policy event revisions are processed in ascending order, successful + events have `processed_at`, and failed events retain a null `processed_at`; +- all v1 reads continue to query Cloud SQL and all v1 responses retain integer + IDs without v2 UUIDs. + +## Application Rollback + +Set `DB_WRITE_POLICY=cloud_sql` or route traffic to the prior application +revision. This immediately removes the Supabase requirement from v1 mutations. +Keep `DB_READ_POLICY=cloud_sql`. Do not delete policies, associations, or +mappings already committed in Supabase; they remain valid for later retries. + +Application rollback does not automatically downgrade either additive source +schema or the v2 schema. If a schema downgrade is separately approved, first +disable native policy traffic and mirroring, verify that no pending Cloud SQL +events or retained v2 data depend on the revisions, and run the reviewed +Alembic downgrades against their confirmed database targets. + +The release workflow restores the exact preceding staging revision with +`DB_WRITE_POLICY=cloud_sql`, verifies the stable service URL, creates another +synthetic v1 policy, confirms that no v2 mapping was created for it, and reads a +policy that was committed to Supabase during activation. It uploads a +90-day-retained JSON artifact containing revision names, timestamps, selector +values, HTTP status summaries, synthetic record identifiers, and non-secret row +counts. The artifact never contains passwords or database URLs. diff --git a/migrations/v1/versions/3d6e8f553ca5_add_saved_policy_mirror_events.py b/migrations/v1/versions/3d6e8f553ca5_add_saved_policy_mirror_events.py new file mode 100644 index 000000000..d393680c9 --- /dev/null +++ b/migrations/v1/versions/3d6e8f553ca5_add_saved_policy_mirror_events.py @@ -0,0 +1,103 @@ +"""add saved policy mirror events + +Revision ID: 3d6e8f553ca5 +Revises: 1914c0422236 +Create Date: 2026-09-01 19:35:29.593649 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "3d6e8f553ca5" +down_revision: Union[str, None] = "1914c0422236" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "user_policy_mirror_events", + sa.Column( + "id", + sa.BigInteger().with_variant(sa.Integer(), "sqlite"), + autoincrement=True, + nullable=False, + ), + sa.Column("country_id", sa.String(length=3), nullable=False), + sa.Column("legacy_user_policy_id", sa.Integer(), nullable=False), + sa.Column("source_revision", sa.BigInteger(), nullable=False), + sa.Column("event_type", sa.String(length=16), nullable=False), + sa.Column("payload_schema_version", sa.SmallInteger(), nullable=False), + sa.Column("payload_json", sa.JSON(), nullable=False), + sa.Column("source_fingerprint_sha256", sa.String(length=64), nullable=False), + sa.Column( + "created_at", + sa.DateTime(), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column("processed_at", sa.DateTime(), nullable=True), + sa.CheckConstraint( + "event_type IN ('create', 'update')", + name="ck_user_policy_mirror_events_event_type", + ), + sa.CheckConstraint( + "length(source_fingerprint_sha256) = 64", + name="ck_user_policy_mirror_events_fingerprint_length", + ), + sa.CheckConstraint( + "payload_schema_version > 0", + name="ck_user_policy_mirror_events_payload_schema_version", + ), + sa.CheckConstraint( + "source_revision > 0", name="ck_user_policy_mirror_events_source_revision" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "country_id", + "legacy_user_policy_id", + "source_revision", + name="uq_user_policy_mirror_events_source_revision", + ), + ) + op.create_index( + "ix_user_policy_mirror_events_pending_age", + "user_policy_mirror_events", + ["processed_at", "created_at"], + unique=False, + ) + op.create_index( + "ix_user_policy_mirror_events_pending_source", + "user_policy_mirror_events", + ["country_id", "legacy_user_policy_id", "processed_at", "source_revision"], + unique=False, + ) + op.add_column( + "user_policies", + sa.Column( + "mirror_revision", + sa.BigInteger(), + server_default=sa.text("0"), + nullable=False, + ), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("user_policies", "mirror_revision") + op.drop_index( + "ix_user_policy_mirror_events_pending_source", + table_name="user_policy_mirror_events", + ) + op.drop_index( + "ix_user_policy_mirror_events_pending_age", + table_name="user_policy_mirror_events", + ) + op.drop_table("user_policy_mirror_events") + # ### end Alembic commands ### diff --git a/migrations/v2/versions/711ec2f0a5a5_migrate_v2_policies.py b/migrations/v2/versions/711ec2f0a5a5_migrate_v2_policies.py new file mode 100644 index 000000000..553eb73f7 --- /dev/null +++ b/migrations/v2/versions/711ec2f0a5a5_migrate_v2_policies.py @@ -0,0 +1,395 @@ +"""migrate v2 policies + +Revision ID: 711ec2f0a5a5 +Revises: 68b4a5ae5dc5 +Create Date: 2026-09-01 16:17:04.855675 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import postgresql + +revision: str = "711ec2f0a5a5" +down_revision: Union[str, None] = "68b4a5ae5dc5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "parameter_values", + "value_json", + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant( + postgresql.JSONB(astext_type=sa.Text()), "postgresql" + ), + existing_nullable=False, + ) + op.create_unique_constraint( + "uq_parameter_values_policy_parameter_start_date", + "parameter_values", + ["policy_id", "parameter_id", "start_date"], + ) + op.create_check_constraint( + op.f("ck_parameter_values_effective_period"), + "parameter_values", + "end_date IS NULL OR end_date >= start_date", + ) + op.add_column( + "policies", + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + ) + op.add_column( + "policies", sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False) + ) + op.add_column( + "policies", sa.Column("canonicalization_version", sa.Integer(), nullable=False) + ) + op.add_column( + "policies", + sa.Column( + "content_hash", sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False + ), + ) + op.create_index( + "ix_policies_country_model", + "policies", + ["country_id", "tax_benefit_model_id"], + unique=False, + ) + op.create_index( + "ix_policies_country_model_version", + "policies", + ["country_id", "tax_benefit_model_version_id"], + unique=False, + ) + op.create_index( + op.f("ix_policies_tax_benefit_model_version_id"), + "policies", + ["tax_benefit_model_version_id"], + unique=False, + ) + op.create_unique_constraint( + "uq_policies_canonicalization_content_hash", + "policies", + ["canonicalization_version", "content_hash"], + ) + op.create_unique_constraint( + "uq_policies_id_country", "policies", ["id", "country_id"] + ) + op.create_foreign_key( + op.f("fk_policies_tax_benefit_model_version_id_tax_benefit_model_versions"), + "policies", + "tax_benefit_model_versions", + ["tax_benefit_model_version_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_check_constraint( + op.f("ck_policies_canonicalization_version"), + "policies", + "canonicalization_version > 0", + ) + op.create_check_constraint( + op.f("ck_policies_content_hash_length"), "policies", "length(content_hash) = 64" + ) + op.create_check_constraint( + op.f("ck_policies_country"), "policies", "country_id IN ('us', 'uk')" + ) + op.drop_column("policies", "name") + op.drop_column("policies", "description") + # Post-generation correction: the generated mapping table preceded the + # composite policy key referenced by its foreign key. + op.create_table( + "legacy_policy_mappings", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + sa.Column("legacy_policy_id", sa.BigInteger(), nullable=False), + sa.Column("policy_id", sa.Uuid(), nullable=False), + sa.Column( + "source_policy_hash", + sqlmodel.sql.sqltypes.AutoString(length=255), + nullable=False, + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", name=op.f("ck_legacy_policy_mappings_country") + ), + sa.ForeignKeyConstraint( + ["policy_id", "country_id"], + ["policies.id", "policies.country_id"], + name="fk_legacy_policy_mappings_policy_country", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_legacy_policy_mappings")), + sa.UniqueConstraint( + "country_id", + "legacy_policy_id", + name="uq_legacy_policy_mappings_country_legacy", + ), + ) + op.create_index( + "ix_legacy_policy_mappings_policy", + "legacy_policy_mappings", + ["policy_id"], + unique=False, + ) + op.add_column( + "user_policies", + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + ) + op.add_column( + "user_policies", + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + ) + op.add_column("user_policies", sa.Column("description", sa.Text(), nullable=True)) + # Post-generation correction: PostgreSQL cannot change the source type + # while its UUID foreign key remains in place. + op.drop_constraint( + op.f("fk_user_policies_user_id_users"), "user_policies", type_="foreignkey" + ) + op.alter_column( + "user_policies", + "user_id", + existing_type=sa.UUID(), + type_=sqlmodel.sql.sqltypes.AutoString(length=255), + existing_nullable=False, + postgresql_using="user_id::text", + ) + op.drop_constraint( + op.f("uq_user_policies_user_policy"), "user_policies", type_="unique" + ) + op.create_index( + "ix_user_policies_country_policy", + "user_policies", + ["country_id", "policy_id"], + unique=False, + ) + op.create_index( + "ix_user_policies_country_user_created_id", + "user_policies", + ["country_id", "user_id", "created_at", "id"], + unique=False, + ) + op.create_unique_constraint( + "uq_user_policies_id_country", "user_policies", ["id", "country_id"] + ) + op.drop_constraint( + op.f("fk_user_policies_policy_id_policies"), "user_policies", type_="foreignkey" + ) + op.create_foreign_key( + "fk_user_policies_policy_country", + "user_policies", + "policies", + ["policy_id", "country_id"], + ["id", "country_id"], + ondelete="RESTRICT", + ) + op.create_check_constraint( + op.f("ck_user_policies_country"), "user_policies", "country_id IN ('us', 'uk')" + ) + op.drop_column("user_policies", "country") + op.drop_column("user_policies", "label") + # Post-generation correction: this mapping likewise depends on the new + # association composite key. + op.create_table( + "legacy_user_policy_mappings", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + sa.Column("legacy_user_policy_id", sa.BigInteger(), nullable=False), + sa.Column("user_policy_id", sa.Uuid(), nullable=False), + sa.Column("fingerprint_version", sa.Integer(), nullable=False), + sa.Column( + "fingerprint_sha256", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=False, + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name=op.f("ck_legacy_user_policy_mappings_country"), + ), + sa.CheckConstraint( + "fingerprint_version > 0", + name=op.f("ck_legacy_user_policy_mappings_fingerprint_version"), + ), + sa.CheckConstraint( + "length(fingerprint_sha256) = 64", + name=op.f("ck_legacy_user_policy_mappings_fingerprint_length"), + ), + sa.ForeignKeyConstraint( + ["user_policy_id", "country_id"], + ["user_policies.id", "user_policies.country_id"], + name="fk_legacy_user_policy_mappings_association_country", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_legacy_user_policy_mappings")), + sa.UniqueConstraint( + "country_id", + "legacy_user_policy_id", + name="uq_legacy_user_policy_mappings_country_legacy", + ), + sa.UniqueConstraint( + "user_policy_id", name="uq_legacy_user_policy_mappings_association" + ), + ) + op.create_index( + "ix_legacy_user_policy_mappings_association", + "legacy_user_policy_mappings", + ["user_policy_id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # Post-generation correction: dependent mappings must be removed before + # their referenced composite keys. + op.drop_index( + "ix_legacy_user_policy_mappings_association", + table_name="legacy_user_policy_mappings", + ) + op.drop_table("legacy_user_policy_mappings") + op.drop_index( + "ix_legacy_policy_mappings_policy", table_name="legacy_policy_mappings" + ) + op.drop_table("legacy_policy_mappings") + op.add_column( + "user_policies", + sa.Column("label", sa.VARCHAR(length=255), autoincrement=False, nullable=True), + ) + op.add_column( + "user_policies", + sa.Column( + "country", sa.VARCHAR(length=16), autoincrement=False, nullable=False + ), + ) + op.drop_constraint(op.f("ck_user_policies_country"), "user_policies", type_="check") + op.drop_constraint( + "fk_user_policies_policy_country", "user_policies", type_="foreignkey" + ) + op.create_foreign_key( + op.f("fk_user_policies_policy_id_policies"), + "user_policies", + "policies", + ["policy_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_constraint("uq_user_policies_id_country", "user_policies", type_="unique") + op.drop_index( + "ix_user_policies_country_user_created_id", table_name="user_policies" + ) + op.drop_index("ix_user_policies_country_policy", table_name="user_policies") + op.create_unique_constraint( + op.f("uq_user_policies_user_policy"), + "user_policies", + ["user_id", "policy_id"], + postgresql_nulls_not_distinct=False, + ) + op.alter_column( + "user_policies", + "user_id", + existing_type=sqlmodel.sql.sqltypes.AutoString(length=255), + type_=sa.UUID(), + existing_nullable=False, + postgresql_using="user_id::uuid", + ) + op.create_foreign_key( + op.f("fk_user_policies_user_id_users"), + "user_policies", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("user_policies", "description") + op.drop_column("user_policies", "name") + op.drop_column("user_policies", "country_id") + op.add_column( + "policies", + sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), + ) + op.add_column( + "policies", + sa.Column("name", sa.VARCHAR(length=255), autoincrement=False, nullable=False), + ) + op.drop_constraint(op.f("ck_policies_country"), "policies", type_="check") + op.drop_constraint( + op.f("ck_policies_content_hash_length"), "policies", type_="check" + ) + op.drop_constraint( + op.f("ck_policies_canonicalization_version"), "policies", type_="check" + ) + op.drop_constraint( + op.f("fk_policies_tax_benefit_model_version_id_tax_benefit_model_versions"), + "policies", + type_="foreignkey", + ) + op.drop_constraint("uq_policies_id_country", "policies", type_="unique") + op.drop_constraint( + "uq_policies_canonicalization_content_hash", "policies", type_="unique" + ) + op.drop_index( + op.f("ix_policies_tax_benefit_model_version_id"), table_name="policies" + ) + op.drop_index("ix_policies_country_model_version", table_name="policies") + op.drop_index("ix_policies_country_model", table_name="policies") + op.drop_column("policies", "content_hash") + op.drop_column("policies", "canonicalization_version") + op.drop_column("policies", "tax_benefit_model_version_id") + op.drop_column("policies", "country_id") + op.drop_constraint( + op.f("ck_parameter_values_effective_period"), "parameter_values", type_="check" + ) + op.drop_constraint( + "uq_parameter_values_policy_parameter_start_date", + "parameter_values", + type_="unique", + ) + op.alter_column( + "parameter_values", + "value_json", + existing_type=sa.JSON().with_variant( + postgresql.JSONB(astext_type=sa.Text()), "postgresql" + ), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=False, + ) + # ### end Alembic commands ### diff --git a/migrations/v2/versions/af34023a728f_map_legacy_users_to_v2_uuids.py b/migrations/v2/versions/af34023a728f_map_legacy_users_to_v2_uuids.py new file mode 100644 index 000000000..43d0cf4b2 --- /dev/null +++ b/migrations/v2/versions/af34023a728f_map_legacy_users_to_v2_uuids.py @@ -0,0 +1,111 @@ +"""map legacy users to v2 uuids + +Revision ID: af34023a728f +Revises: c21c4a807a49 +Create Date: 2026-09-01 22:06:28.086723 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "af34023a728f" +down_revision: Union[str, None] = "c21c4a807a49" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "legacy_user_mappings", + sa.Column( + "legacy_user_id", + sqlmodel.sql.sqltypes.AutoString(length=255), + nullable=False, + ), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name=op.f("fk_legacy_user_mappings_user_id_users"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("legacy_user_id", name=op.f("pk_legacy_user_mappings")), + sa.UniqueConstraint("user_id", name="uq_legacy_user_mappings_user_id"), + ) + op.create_index( + op.f("ix_legacy_user_mappings_user_id"), + "legacy_user_mappings", + ["user_id"], + unique=False, + ) + op.alter_column( + "user_policies", + "user_id", + existing_type=sa.VARCHAR(length=255), + type_=sa.Uuid(), + existing_nullable=False, + # Post-generation correction: PostgreSQL requires an explicit + # cast when restoring the UUID association foreign key. + postgresql_using="user_id::uuid", + ) + op.create_foreign_key( + op.f("fk_user_policies_user_id_users"), + "user_policies", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.alter_column( + "users", "first_name", existing_type=sa.VARCHAR(length=255), nullable=True + ) + op.alter_column( + "users", "last_name", existing_type=sa.VARCHAR(length=255), nullable=True + ) + op.alter_column( + "users", "email", existing_type=sa.VARCHAR(length=320), nullable=True + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "users", "email", existing_type=sa.VARCHAR(length=320), nullable=False + ) + op.alter_column( + "users", "last_name", existing_type=sa.VARCHAR(length=255), nullable=False + ) + op.alter_column( + "users", "first_name", existing_type=sa.VARCHAR(length=255), nullable=False + ) + op.drop_constraint( + op.f("fk_user_policies_user_id_users"), "user_policies", type_="foreignkey" + ) + op.alter_column( + "user_policies", + "user_id", + existing_type=sa.Uuid(), + type_=sa.VARCHAR(length=255), + existing_nullable=False, + # Post-generation correction: PostgreSQL requires an explicit + # cast for the reversible UUID-to-text downgrade. + postgresql_using="user_id::text", + ) + op.drop_index( + op.f("ix_legacy_user_mappings_user_id"), table_name="legacy_user_mappings" + ) + op.drop_table("legacy_user_mappings") + # ### end Alembic commands ### diff --git a/migrations/v2/versions/c21c4a807a49_track_saved_policy_mirror_revisions.py b/migrations/v2/versions/c21c4a807a49_track_saved_policy_mirror_revisions.py new file mode 100644 index 000000000..91042f206 --- /dev/null +++ b/migrations/v2/versions/c21c4a807a49_track_saved_policy_mirror_revisions.py @@ -0,0 +1,49 @@ +"""track saved policy mirror revisions + +Revision ID: c21c4a807a49 +Revises: 711ec2f0a5a5 +Create Date: 2026-09-01 19:35:37.381580 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "c21c4a807a49" +down_revision: Union[str, None] = "711ec2f0a5a5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "legacy_user_policy_mappings", + sa.Column( + "last_applied_source_revision", + sa.BigInteger(), + server_default="0", + nullable=False, + ), + ) + op.create_check_constraint( + op.f("ck_legacy_user_policy_mappings_source_revision"), + "legacy_user_policy_mappings", + "last_applied_source_revision >= 0", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("ck_legacy_user_policy_mappings_source_revision"), + "legacy_user_policy_mappings", + type_="check", + ) + op.drop_column("legacy_user_policy_mappings", "last_applied_source_revision") + # ### end Alembic commands ### diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index 171cf0c61..bcc9802ed 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -19,7 +19,11 @@ from policyengine_api.fastapi_routes.specification import ( build_specification_router, ) -from policyengine_api.fastapi_routes.v2_metadata import build_v2_metadata_router +from policyengine_api.fastapi_routes.v2.errors import ( + V2RequestTooLargeError, + v2_error_response, +) +from policyengine_api.fastapi_routes.v2.routes import build_v2_router from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, @@ -31,29 +35,17 @@ generate_request_id, ) from starlette.datastructures import MutableHeaders +from starlette.middleware.cors import CORSMiddleware from starlette.middleware.gzip import GZipMiddleware from starlette.responses import PlainTextResponse, Response +from starlette.types import ASGIApp -def _add_vary_origin(response) -> None: - vary = response.headers.get("Vary") - if vary is None: - response.headers["Vary"] = "Origin" - return - if "origin" not in {value.strip().lower() for value in vary.split(",")}: - response.headers["Vary"] = f"{vary}, Origin" - - -def _apply_shared_response_headers( - request: Request, +def _apply_request_id_header( response: Response, request_id: str, ) -> None: response.headers[REQUEST_ID_HEADER] = request_id - origin = request.headers.get("origin") - if origin and "access-control-allow-origin" not in response.headers: - response.headers["Access-Control-Allow-Origin"] = origin - _add_vary_origin(response) def create_asgi_app( @@ -62,7 +54,7 @@ def create_asgi_app( route_settings: RouteImplementationSettings | None = None, dependencies: NativeRouteDependencies | None = None, shutdown_callback: Callable[[], None] | None = None, -) -> FastAPI: +) -> ASGIApp: """Create the Stage 2 FastAPI shell around the existing Flask app.""" if route_settings is None: @@ -104,7 +96,7 @@ async def add_headers_to_unhandled_errors( "policyengine_request_id", request.headers.get(REQUEST_ID_HEADER) or generate_request_id(), ) - _apply_shared_response_headers(request, response, request_id) + _apply_request_id_header(response, request_id) return response @app.exception_handler(RequestValidationError) @@ -113,15 +105,18 @@ async def typed_v2_request_validation_error( error: RequestValidationError, ) -> Response: if request.url.path.startswith("/v2/"): - from policyengine_api.fastapi_routes.v2_metadata_common import ( - error_response, - ) - - return error_response(422, "Invalid v2 metadata request") + return v2_error_response(422, "Invalid API v2 request") return await request_validation_exception_handler(request, error) + @app.exception_handler(V2RequestTooLargeError) + async def oversized_v2_request( + _request: Request, + error: V2RequestTooLargeError, + ) -> Response: + return v2_error_response(413, str(error)) + @app.middleware("http") - async def add_cors_for_native_routes(request, call_next): + async def add_request_context_and_migration_logging(request, call_next): started_at = time.time() request_id = request.headers.get(REQUEST_ID_HEADER) or generate_request_id() MutableHeaders(scope=request.scope)[REQUEST_ID_HEADER] = request_id @@ -153,14 +148,14 @@ def log_native_route(status_code: int) -> None: except Exception: log_native_route(500) raise - _apply_shared_response_headers(request, response, request_id) + _apply_request_id_header(response, request_id) log_native_route(response.status_code) return response finally: _asgi_request_id.reset(context_token) app.include_router(build_core_health_router(dependencies)) - app.include_router(build_v2_metadata_router(dependencies)) + app.include_router(build_v2_router(dependencies)) if route_settings.health is RouteImplementation.FASTAPI_NATIVE: app.include_router(build_readiness_router(dependencies)) if route_settings.specification is RouteImplementation.FASTAPI_NATIVE: @@ -169,4 +164,14 @@ def log_native_route(status_code: int) -> None: app.include_router(build_metadata_router(dependencies)) app.mount("/", WSGIMiddleware(wsgi_app)) - return app + # The public API already permits every web origin. Use the standard ASGI + # implementation while preserving the existing reflected-origin response. + return CORSMiddleware( + app=app, + allow_origin_regex=".*", + allow_methods=["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"], + allow_headers=["*"], + expose_headers=[REQUEST_ID_HEADER], + allow_credentials=False, + max_age=600, + ) diff --git a/policyengine_api/data/v1_models.py b/policyengine_api/data/v1_models.py index 6e848d106..df3b58362 100644 --- a/policyengine_api/data/v1_models.py +++ b/policyengine_api/data/v1_models.py @@ -12,9 +12,12 @@ from sqlalchemy import ( BigInteger, CHAR, + CheckConstraint, DateTime, + Index, Integer, JSON, + SmallInteger, String, Text, UniqueConstraint, @@ -127,6 +130,73 @@ class UserPolicy(V1Base): updated_date: Mapped[int] = mapped_column(BigInteger) budgetary_impact: Mapped[str | None] = mapped_column(String(255)) type: Mapped[str | None] = mapped_column(String(255)) + mirror_revision: Mapped[int] = mapped_column( + BigInteger, + nullable=False, + default=0, + server_default=text("0"), + ) + + +class UserPolicyMirrorEvent(V1Base): + """Ordered, durable input for synchronous saved-policy mirroring.""" + + __tablename__ = "user_policy_mirror_events" + __table_args__ = ( + UniqueConstraint( + "country_id", + "legacy_user_policy_id", + "source_revision", + name="uq_user_policy_mirror_events_source_revision", + ), + CheckConstraint( + "source_revision > 0", + name="ck_user_policy_mirror_events_source_revision", + ), + CheckConstraint( + "event_type IN ('create', 'update')", + name="ck_user_policy_mirror_events_event_type", + ), + CheckConstraint( + "payload_schema_version > 0", + name="ck_user_policy_mirror_events_payload_schema_version", + ), + CheckConstraint( + "length(source_fingerprint_sha256) = 64", + name="ck_user_policy_mirror_events_fingerprint_length", + ), + Index( + "ix_user_policy_mirror_events_pending_source", + "country_id", + "legacy_user_policy_id", + "processed_at", + "source_revision", + ), + Index( + "ix_user_policy_mirror_events_pending_age", + "processed_at", + "created_at", + ), + ) + + id: Mapped[int] = mapped_column( + BigInteger().with_variant(Integer, "sqlite"), + primary_key=True, + autoincrement=True, + ) + country_id: Mapped[str] = mapped_column(String(3)) + legacy_user_policy_id: Mapped[int] = mapped_column(Integer) + source_revision: Mapped[int] = mapped_column(BigInteger) + event_type: Mapped[str] = mapped_column(String(16)) + payload_schema_version: Mapped[int] = mapped_column(SmallInteger) + payload_json: Mapped[Any] = mapped_column(JSON) + source_fingerprint_sha256: Mapped[str] = mapped_column(String(64)) + created_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=text("CURRENT_TIMESTAMP"), + ) + processed_at: Mapped[datetime | None] = mapped_column(DateTime) class UserProfile(V1Base): diff --git a/policyengine_api/data/v2/catalog/catalog_selection.py b/policyengine_api/data/v2/catalog/catalog_selection.py index 7014210c9..e4a32fb3c 100644 --- a/policyengine_api/data/v2/catalog/catalog_selection.py +++ b/policyengine_api/data/v2/catalog/catalog_selection.py @@ -14,7 +14,7 @@ ) -SUPPORTED_PREVIEW_COUNTRIES = frozenset({"us", "uk"}) +SUPPORTED_V2_COUNTRY_IDS = frozenset({"us", "uk"}) class MetadataCatalogUnavailableError(RuntimeError): @@ -76,7 +76,7 @@ def select_catalog( ) -> SelectedCatalog: """Select one country catalog using the requested or running package version.""" - if country_id not in SUPPORTED_PREVIEW_COUNTRIES: + if country_id not in SUPPORTED_V2_COUNTRY_IDS: raise UnsupportedPreviewCountryError(country_id) explicit_version = policyengine_version is not None selected_version = ( diff --git a/policyengine_api/data/v2/catalog/dataset_query.py b/policyengine_api/data/v2/catalog/dataset_query.py deleted file mode 100644 index 755d5b006..000000000 --- a/policyengine_api/data/v2/catalog/dataset_query.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Logical input-dataset metadata queries.""" - -from __future__ import annotations - -from uuid import UUID - -from sqlmodel import select -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, - MetadataResourceNotFoundError, - page_result, - query_rows, -) -from policyengine_api.data.v2.catalog.schemas import ( - MetadataDataset, - MetadataDetailResult, - MetadataPageResult, -) -from policyengine_api.data.v2.models import Dataset - - -def _dataset(dataset: Dataset) -> MetadataDataset: - return MetadataDataset( - id=dataset.id, - name=dataset.name, - description=dataset.description, - year=dataset.year, - ) - - -class DatasetQueryMethods(MetadataQueryContext): - """Route-facing logical input-dataset query methods.""" - - def list_datasets( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataDataset]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - rows = query_rows( - self._session, - select(Dataset) - .where( - Dataset.tax_benefit_model_version_id == selected.model_version.id, - Dataset.is_output_dataset.is_(False), - Dataset.storage_path.is_(None), - ) - .order_by(Dataset.name) - .offset(offset) - .limit(limit + 1), - ) - return page_result( - selected, - [_dataset(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_dataset( - self, - country_id: str, - dataset_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataDataset]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Dataset).where( - Dataset.id == dataset_id, - Dataset.tax_benefit_model_version_id == selected.model_version.id, - Dataset.is_output_dataset.is_(False), - Dataset.storage_path.is_(None), - ), - ) - if not rows: - raise MetadataResourceNotFoundError(f"dataset {dataset_id} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_dataset(rows[0]), - ) diff --git a/policyengine_api/data/v2/catalog/model_query.py b/policyengine_api/data/v2/catalog/model_query.py deleted file mode 100644 index 17a0e130b..000000000 --- a/policyengine_api/data/v2/catalog/model_query.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Tax-benefit model and model-version metadata queries.""" - -from __future__ import annotations - -from uuid import UUID - -from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, - MetadataResourceNotFoundError, - page_result, -) -from policyengine_api.data.v2.catalog.schemas import ( - MetadataDetailResult, - MetadataModel, - MetadataModelSelectionResult, - MetadataModelVersionDetail, - MetadataPageResult, -) - - -def _model(selected: SelectedCatalog) -> MetadataModel: - return MetadataModel( - id=selected.model.id, - name=selected.model.name, - description=selected.model_version.description, - ) - - -def _model_version(selected: SelectedCatalog) -> MetadataModelVersionDetail: - return MetadataModelVersionDetail( - id=selected.model_version.id, - model_id=selected.model.id, - version=selected.model_version.version, - description=selected.model_version.description, - current_law_id=selected.model_version.current_law_id, - metadata_time_periods=selected.model_version.metadata_time_periods, - ) - - -class ModelQueryMethods(MetadataQueryContext): - """Route-facing model and model-version query methods.""" - - def list_models( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataModel]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - rows = [_model(selected)] if offset == 0 else [] - return page_result(selected, rows, offset=offset, limit=limit) - - def get_model( - self, - country_id: str, - model_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataModel]: - selected = self.select_catalog(country_id, policyengine_version) - if selected.model.id != model_id: - raise MetadataResourceNotFoundError(f"model {model_id} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_model(selected), - ) - - def get_model_by_country( - self, - country_id: str, - policyengine_version: str | None = None, - ) -> MetadataModelSelectionResult: - selected = self.select_catalog(country_id, policyengine_version) - return MetadataModelSelectionResult( - policyengine_version=selected.policyengine_version, - model=_model(selected), - model_version=_model_version(selected), - ) - - def list_model_versions( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataModelVersionDetail]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - rows = [_model_version(selected)] if offset == 0 else [] - return page_result(selected, rows, offset=offset, limit=limit) - - def get_model_version( - self, - country_id: str, - version_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataModelVersionDetail]: - selected = self.select_catalog(country_id, policyengine_version) - if selected.model_version.id != version_id: - raise MetadataResourceNotFoundError( - f"model version {version_id} was not found" - ) - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_model_version(selected), - ) diff --git a/policyengine_api/data/v2/catalog/parameter_query.py b/policyengine_api/data/v2/catalog/parameter_query.py deleted file mode 100644 index c3807f1f2..000000000 --- a/policyengine_api/data/v2/catalog/parameter_query.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Parameter, parameter-tree, and canonical parameter-value queries.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from uuid import UUID - -import sqlalchemy as sa -from sqlmodel import select -from policyengine_api.data.v2.catalog.parameter_tree_query import ( - parameter_children_from_rows, - parameter_children_query, -) -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, - MetadataResourceNotFoundError, - escape_like, - page_result, - query_rows, -) -from policyengine_api.data.v2.catalog.schemas import ( - MetadataCanonicalParameterValue, - MetadataDetailResult, - MetadataPageResult, - MetadataParameterChild, - MetadataParameterSummary, -) -from policyengine_api.data.v2.models import Parameter, ParameterValue - - -def _parameter(parameter: Parameter) -> MetadataParameterSummary: - return MetadataParameterSummary( - id=parameter.id, - name=parameter.name, - label=parameter.label, - description=parameter.description, - data_type=parameter.data_type, - unit=parameter.unit, - ) - - -def _parameter_value(value: ParameterValue) -> MetadataCanonicalParameterValue: - return MetadataCanonicalParameterValue( - id=value.id, - parameter_id=value.parameter_id, - value=value.value_json, - start_date=value.start_date, - end_date=value.end_date, - ) - - -def _utc_day_start(selected_time: datetime) -> datetime: - if selected_time.tzinfo is None: - selected_time = selected_time.replace(tzinfo=timezone.utc) - return selected_time.astimezone(timezone.utc).replace( - hour=0, - minute=0, - second=0, - microsecond=0, - ) - - -class ParameterQueryMethods(MetadataQueryContext): - """Route-facing parameter, tree, and canonical-value query methods.""" - - def list_parameters( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - search: str | None = None, - ) -> MetadataPageResult[MetadataParameterSummary]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - statement = select(Parameter).where( - Parameter.tax_benefit_model_version_id == selected.model_version.id - ) - if search: - pattern = f"%{escape_like(search)}%" - statement = statement.where( - sa.or_( - Parameter.name.ilike(pattern, escape="\\"), - Parameter.label.ilike(pattern, escape="\\"), - Parameter.description.ilike(pattern, escape="\\"), - ) - ) - rows = query_rows( - self._session, - statement.order_by(Parameter.name).offset(offset).limit(limit + 1), - ) - return page_result( - selected, - [_parameter(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_parameter( - self, - country_id: str, - parameter_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataParameterSummary]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Parameter).where( - Parameter.id == parameter_id, - Parameter.tax_benefit_model_version_id == selected.model_version.id, - ), - ) - if not rows: - raise MetadataResourceNotFoundError( - f"parameter {parameter_id} was not found" - ) - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_parameter(rows[0]), - ) - - def list_parameter_children( - self, - country_id: str, - policyengine_version: str | None = None, - *, - parent_path: str = "", - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataParameterChild]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - rows = query_rows( - self._session, - parameter_children_query( - model_version_id=selected.model_version.id, - parent_path=parent_path, - dialect=self._session.get_bind().dialect.name, - offset=offset, - limit=limit, - ), - ) - return page_result( - selected, - parameter_children_from_rows(rows), - offset=offset, - limit=limit, - ) - - def list_parameter_values( - self, - country_id: str, - policyengine_version: str | None = None, - *, - parameter_id: UUID | None = None, - current: bool = False, - offset: int = 0, - limit: int = 100, - now: datetime | None = None, - ) -> MetadataPageResult[MetadataCanonicalParameterValue]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - statement = ( - select(ParameterValue) - .join(Parameter, Parameter.id == ParameterValue.parameter_id) - .where( - Parameter.tax_benefit_model_version_id == selected.model_version.id, - ParameterValue.policy_id.is_(None), - ParameterValue.dynamic_id.is_(None), - ) - ) - if parameter_id is not None: - statement = statement.where(ParameterValue.parameter_id == parameter_id) - if current: - selected_day = _utc_day_start(now or datetime.now(timezone.utc)) - statement = statement.where( - ParameterValue.start_date <= selected_day, - sa.or_( - ParameterValue.end_date.is_(None), - ParameterValue.end_date >= selected_day, - ), - ) - rows = query_rows( - self._session, - statement.order_by( - Parameter.name, - ParameterValue.start_date.desc(), - ParameterValue.id, - ) - .offset(offset) - .limit(limit + 1), - ) - return page_result( - selected, - [_parameter_value(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_parameter_value( - self, - country_id: str, - value_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataCanonicalParameterValue]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(ParameterValue) - .join(Parameter, Parameter.id == ParameterValue.parameter_id) - .where( - ParameterValue.id == value_id, - Parameter.tax_benefit_model_version_id == selected.model_version.id, - ParameterValue.policy_id.is_(None), - ParameterValue.dynamic_id.is_(None), - ), - ) - if not rows: - raise MetadataResourceNotFoundError( - f"parameter value {value_id} was not found" - ) - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_parameter_value(rows[0]), - ) diff --git a/policyengine_api/data/v2/catalog/parameter_tree_query.py b/policyengine_api/data/v2/catalog/parameter_tree_query.py deleted file mode 100644 index cc4c856a7..000000000 --- a/policyengine_api/data/v2/catalog/parameter_tree_query.py +++ /dev/null @@ -1,167 +0,0 @@ -"""SQL query and row conversion for direct parameter-tree children.""" - -from __future__ import annotations - -from uuid import UUID - -import sqlalchemy as sa -from sqlmodel import select - -from policyengine_api.data.v2.catalog.schemas import ( - MetadataParameterChild, - MetadataParameterSummary, -) -from policyengine_api.data.v2.models import Parameter, ParameterNode - - -def _escaped_like(value: str) -> str: - return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - - -def _path_segment(remainder: object, dialect: str) -> object: - dot_position = ( - sa.func.instr(remainder, ".") - if dialect == "sqlite" - else sa.func.strpos(remainder, ".") - ) - return sa.case( - (dot_position > 0, sa.func.substr(remainder, 1, dot_position - 1)), - else_=remainder, - ) - - -def _child_path(column: object, prefix: str, dialect: str) -> object: - remainder = sa.func.substr(column, len(prefix) + 1) - return sa.literal(prefix) + _path_segment(remainder, dialect) - - -def _direct_child_path( - column: object, - parent_path: object, - dialect: str, -) -> object: - remainder = sa.func.substr(column, sa.func.length(parent_path) + 2) - return parent_path + "." + _path_segment(remainder, dialect) - - -def _has_path_prefix(column: object, parent_path: object) -> object: - return ( - sa.func.substr(column, 1, sa.func.length(parent_path) + 1) == parent_path + "." - ) - - -def parameter_children_query( - *, - model_version_id: UUID, - parent_path: str, - dialect: str, - offset: int, - limit: int, -) -> object: - """Build one bounded query for a parameter path's direct children.""" - - prefix = f"{parent_path}." if parent_path else "" - escaped_prefix = _escaped_like(prefix) - node_child_path = _child_path(ParameterNode.name, prefix, dialect) - parameter_child_path = _child_path(Parameter.name, prefix, dialect) - paths = sa.union( - select(node_child_path.label("path")).where( - ParameterNode.tax_benefit_model_version_id == model_version_id, - ParameterNode.name.like(f"{escaped_prefix}%", escape="\\"), - ), - select(parameter_child_path.label("path")).where( - Parameter.tax_benefit_model_version_id == model_version_id, - Parameter.name.like(f"{escaped_prefix}%", escape="\\"), - ), - ).subquery() - direct_child_paths = sa.union( - select( - _direct_child_path( - ParameterNode.name, - paths.c.path, - dialect, - ).label("path") - ) - .where( - ParameterNode.tax_benefit_model_version_id == model_version_id, - _has_path_prefix(ParameterNode.name, paths.c.path), - ) - .correlate(paths), - select( - _direct_child_path( - Parameter.name, - paths.c.path, - dialect, - ).label("path") - ) - .where( - Parameter.tax_benefit_model_version_id == model_version_id, - _has_path_prefix(Parameter.name, paths.c.path), - ) - .correlate(paths), - ).subquery() - direct_child_count = ( - select(sa.func.count()) - .select_from(direct_child_paths) - .correlate(paths) - .scalar_subquery() - ) - is_node = sa.or_(direct_child_count > 0, Parameter.id.is_(None)) - return ( - select( - paths.c.path, - sa.func.coalesce(ParameterNode.label, Parameter.label).label("label"), - sa.case((is_node, "node"), else_="parameter").label("type"), - sa.case((is_node, direct_child_count), else_=None).label("child_count"), - Parameter.id.label("parameter_id"), - Parameter.label.label("parameter_label"), - Parameter.description.label("parameter_description"), - Parameter.data_type.label("parameter_data_type"), - Parameter.unit.label("parameter_unit"), - ) - .select_from( - paths.outerjoin( - ParameterNode, - sa.and_( - ParameterNode.name == paths.c.path, - ParameterNode.tax_benefit_model_version_id == model_version_id, - ), - ).outerjoin( - Parameter, - sa.and_( - Parameter.name == paths.c.path, - Parameter.tax_benefit_model_version_id == model_version_id, - ), - ) - ) - .order_by(paths.c.path) - .offset(offset) - .limit(limit + 1) - ) - - -def parameter_children_from_rows(rows: list) -> list[MetadataParameterChild]: - """Convert parameter-tree query rows into typed direct-child records.""" - - items = [] - for row in rows: - parameter = None - if row.type == "parameter": - parameter = MetadataParameterSummary( - id=row.parameter_id, - name=row.path, - label=row.parameter_label, - description=row.parameter_description, - data_type=row.parameter_data_type, - unit=row.parameter_unit, - ) - items.append( - MetadataParameterChild( - path=row.path, - label=row.label or row.path.rsplit(".", 1)[-1], - type=row.type, - child_count=row.child_count, - parameter=parameter, - ) - ) - return items diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 3e59306cf..2f12581a0 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -32,7 +32,7 @@ ) -EXPECTED_ALEMBIC_REVISION = "68b4a5ae5dc5" +EXPECTED_ALEMBIC_REVISION = "af34023a728f" # Stable application-defined PostgreSQL lock ID shared by all v2 catalog publishers. PUBLICATION_ADVISORY_LOCK_KEY = 8_629_020_026_090_001 diff --git a/policyengine_api/data/v2/catalog/query.py b/policyengine_api/data/v2/catalog/query.py deleted file mode 100644 index 8e24ffa60..000000000 --- a/policyengine_api/data/v2/catalog/query.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Public read-only v2 metadata query service.""" - -from __future__ import annotations - -from policyengine_api.data.v2.catalog.catalog_selection import ( - InvalidPolicyEngineVersionError, - MetadataCatalogUnavailableError, - MetadataCatalogVersionNotFoundError, - UnsupportedPreviewCountryError, - validate_policyengine_version, -) -from policyengine_api.data.v2.catalog.dataset_query import DatasetQueryMethods -from policyengine_api.data.v2.catalog.model_query import ModelQueryMethods -from policyengine_api.data.v2.catalog.parameter_query import ParameterQueryMethods -from policyengine_api.data.v2.catalog.query_support import ( - InvalidMetadataPageError, - MetadataResourceNotFoundError, - validate_metadata_page, -) -from policyengine_api.data.v2.catalog.region_query import RegionQueryMethods -from policyengine_api.data.v2.catalog.variable_query import VariableQueryMethods - - -__all__ = [ - "InvalidMetadataPageError", - "InvalidPolicyEngineVersionError", - "MetadataCatalogUnavailableError", - "MetadataCatalogVersionNotFoundError", - "MetadataResourceNotFoundError", - "UnsupportedPreviewCountryError", - "V2MetadataQueryService", - "validate_metadata_page", - "validate_policyengine_version", -] - - -class V2MetadataQueryService( - ModelQueryMethods, - VariableQueryMethods, - ParameterQueryMethods, - DatasetQueryMethods, - RegionQueryMethods, -): - """Combine the resource-specific query methods into the route-facing API.""" diff --git a/policyengine_api/data/v2/catalog/query_support.py b/policyengine_api/data/v2/catalog/query_support.py deleted file mode 100644 index 8dd0b49e5..000000000 --- a/policyengine_api/data/v2/catalog/query_support.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Shared execution and pagination for v2 metadata resource queries.""" - -from __future__ import annotations - -from typing import TypeVar - -from sqlalchemy.exc import SQLAlchemyError -from sqlmodel import Session - -from policyengine_api.data.v2.catalog.catalog_selection import ( - MetadataCatalogUnavailableError, - SelectedCatalog, - select_catalog as select_metadata_catalog, - validate_policyengine_version, -) -from policyengine_api.data.v2.catalog.schemas import MetadataPageResult - - -class MetadataResourceNotFoundError(LookupError): - """Raised when a selected catalog does not contain a requested resource.""" - - -class InvalidMetadataPageError(ValueError): - """Raised when collection pagination is outside the documented bounds.""" - - -ResourceT = TypeVar("ResourceT") - - -class MetadataQueryContext: - """Own the session and exact catalog selection shared by resource queries.""" - - def __init__(self, session: Session, *, running_policyengine_version: str): - self._session = session - self._running_policyengine_version = validate_policyengine_version( - running_policyengine_version - ) - - def close(self) -> None: - """Close the request-owned read session.""" - - self._session.close() - - def select_catalog( - self, - country_id: str, - policyengine_version: str | None = None, - ) -> SelectedCatalog: - """Select exactly one initialized country catalog.""" - - return select_metadata_catalog( - self._session, - country_id=country_id, - running_policyengine_version=self._running_policyengine_version, - policyengine_version=policyengine_version, - ) - - def _select_paginated_catalog( - self, - country_id: str, - policyengine_version: str | None, - *, - offset: int, - limit: int, - ) -> SelectedCatalog: - validate_metadata_page(offset, limit) - return self.select_catalog(country_id, policyengine_version) - - -def page_result( - selected: SelectedCatalog, - rows: list[ResourceT], - *, - offset: int, - limit: int, -) -> MetadataPageResult[ResourceT]: - """Return one bounded response page from a limit-plus-one query.""" - - return MetadataPageResult( - policyengine_version=selected.policyengine_version, - items=rows[:limit], - offset=offset, - limit=limit, - has_more=len(rows) > limit, - ) - - -def validate_metadata_page(offset: int, limit: int) -> tuple[int, int]: - """Validate the shared v2 metadata collection bounds.""" - - if offset < 0: - raise InvalidMetadataPageError("offset must be at least 0") - if not 1 <= limit <= 500: - raise InvalidMetadataPageError("limit must be between 1 and 500") - return offset, limit - - -def query_rows(session: Session, statement: object) -> list: - """Execute one read statement and translate database failures.""" - - try: - return list(session.exec(statement).all()) - except SQLAlchemyError as error: - raise MetadataCatalogUnavailableError( - "the v2 metadata catalog cannot be queried" - ) from error - - -def escape_like(value: str) -> str: - """Escape SQL LIKE wildcard characters in a literal search value.""" - - return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") diff --git a/policyengine_api/data/v2/catalog/region_query.py b/policyengine_api/data/v2/catalog/region_query.py deleted file mode 100644 index 8ac0f3417..000000000 --- a/policyengine_api/data/v2/catalog/region_query.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Region and economy-option metadata queries.""" - -from __future__ import annotations - -from uuid import UUID - -from sqlmodel import select - -from policyengine_api.dataset_display import get_dataset_display_label -from policyengine_api.data.v2.catalog.catalog_selection import ( - MetadataCatalogUnavailableError, -) -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, - MetadataResourceNotFoundError, - page_result, - query_rows, -) -from policyengine_api.data.v2.catalog.schemas import ( - MetadataDatasetOption, - MetadataDetailResult, - MetadataEconomyOptionsResult, - MetadataPageResult, - MetadataRegion, - MetadataRegionOption, - MetadataTimePeriodOption, -) -from policyengine_api.data.v2.models import Dataset, Region - - -def _region(region: Region) -> MetadataRegion: - return MetadataRegion( - id=region.id, - code=region.code, - label=region.label, - region_type=region.region_type.value, - requires_filter=region.requires_filter, - filter_field=region.filter_field, - filter_value=region.filter_value, - filter_strategy=region.filter_strategy, - parent_code=region.parent_code, - state_code=region.state_code, - state_name=region.state_name, - default_dataset_id=region.default_dataset_id, - ) - - -class RegionQueryMethods(MetadataQueryContext): - """Route-facing region and economy-option query methods.""" - - def list_regions( - self, - country_id: str, - policyengine_version: str | None = None, - *, - region_type: str | None = None, - offset: int = 0, - limit: int = 100, - ) -> MetadataPageResult[MetadataRegion]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - statement = select(Region).where( - Region.tax_benefit_model_version_id == selected.model_version.id - ) - if region_type is not None: - statement = statement.where(Region.region_type == region_type) - rows = query_rows( - self._session, - statement.order_by(Region.code).offset(offset).limit(limit + 1), - ) - return page_result( - selected, - [_region(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_region( - self, - country_id: str, - region_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataRegion]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Region).where( - Region.id == region_id, - Region.tax_benefit_model_version_id == selected.model_version.id, - ), - ) - if not rows: - raise MetadataResourceNotFoundError(f"region {region_id} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_region(rows[0]), - ) - - def get_region_by_code( - self, - country_id: str, - region_code: str, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataRegion]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Region).where( - Region.code == region_code, - Region.tax_benefit_model_version_id == selected.model_version.id, - ), - ) - if not rows: - raise MetadataResourceNotFoundError(f"region {region_code!r} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_region(rows[0]), - ) - - def get_economy_options( - self, - country_id: str, - policyengine_version: str | None = None, - ) -> MetadataEconomyOptionsResult: - selected = self.select_catalog(country_id, policyengine_version) - regions = query_rows( - self._session, - select(Region) - .where(Region.tax_benefit_model_version_id == selected.model_version.id) - .order_by(Region.code), - ) - national_region = next( - (region for region in regions if region.code == country_id), - None, - ) - if national_region is None: - raise MetadataCatalogUnavailableError( - f"the {country_id} national v2 region is absent" - ) - datasets = query_rows( - self._session, - select(Dataset).where( - Dataset.id == national_region.default_dataset_id, - Dataset.tax_benefit_model_version_id == selected.model_version.id, - Dataset.is_output_dataset.is_(False), - Dataset.storage_path.is_(None), - ), - ) - if len(datasets) != 1: - raise MetadataCatalogUnavailableError( - f"the {country_id} national v2 dataset is absent" - ) - time_periods = selected.model_version.metadata_time_periods - if ( - not isinstance(selected.model_version.current_law_id, int) - or not isinstance(time_periods, list) - or not time_periods - or any(not isinstance(year, int) for year in time_periods) - ): - raise MetadataCatalogUnavailableError( - f"the {country_id} v2 model-version options are incomplete" - ) - national_dataset = datasets[0] - return MetadataEconomyOptionsResult( - policyengine_version=selected.policyengine_version, - current_law_id=selected.model_version.current_law_id, - region=[ - MetadataRegionOption( - name=region.code, - label=region.label, - type=region.region_type.value, - ) - for region in regions - ], - time_period=[ - MetadataTimePeriodOption(name=year, label=str(year)) - for year in time_periods - ], - datasets=[ - MetadataDatasetOption( - name=national_dataset.name, - label=get_dataset_display_label(national_dataset.name), - ) - ], - ) diff --git a/policyengine_api/data/v2/catalog/variable_query.py b/policyengine_api/data/v2/catalog/variable_query.py deleted file mode 100644 index 6f6c79dd9..000000000 --- a/policyengine_api/data/v2/catalog/variable_query.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Variable metadata queries.""" - -from __future__ import annotations - -from uuid import UUID - -import sqlalchemy as sa -from sqlmodel import select -from policyengine_api.data.v2.catalog.query_support import ( - MetadataQueryContext, - MetadataResourceNotFoundError, - escape_like, - page_result, - query_rows, -) -from policyengine_api.data.v2.catalog.schemas import ( - MetadataDetailResult, - MetadataPageResult, - MetadataVariable, -) -from policyengine_api.data.v2.models import Variable - - -def _variable(variable: Variable) -> MetadataVariable: - return MetadataVariable( - id=variable.id, - name=variable.name, - label=variable.label, - entity=variable.entity, - description=variable.description, - data_type=variable.data_type, - possible_values=variable.possible_values, - default_value=variable.default_value, - adds=variable.adds, - subtracts=variable.subtracts, - ) - - -class VariableQueryMethods(MetadataQueryContext): - """Route-facing variable query methods.""" - - def list_variables( - self, - country_id: str, - policyengine_version: str | None = None, - *, - offset: int = 0, - limit: int = 100, - search: str | None = None, - ) -> MetadataPageResult[MetadataVariable]: - selected = self._select_paginated_catalog( - country_id, - policyengine_version, - offset=offset, - limit=limit, - ) - statement = select(Variable).where( - Variable.tax_benefit_model_version_id == selected.model_version.id - ) - if search: - pattern = f"%{escape_like(search)}%" - statement = statement.where( - sa.or_( - Variable.name.ilike(pattern, escape="\\"), - Variable.label.ilike(pattern, escape="\\"), - Variable.description.ilike(pattern, escape="\\"), - ) - ) - rows = query_rows( - self._session, - statement.order_by(Variable.name).offset(offset).limit(limit + 1), - ) - return page_result( - selected, - [_variable(row) for row in rows], - offset=offset, - limit=limit, - ) - - def get_variable( - self, - country_id: str, - variable_id: UUID, - policyengine_version: str | None = None, - ) -> MetadataDetailResult[MetadataVariable]: - selected = self.select_catalog(country_id, policyengine_version) - rows = query_rows( - self._session, - select(Variable).where( - Variable.id == variable_id, - Variable.tax_benefit_model_version_id == selected.model_version.id, - ), - ) - if not rows: - raise MetadataResourceNotFoundError(f"variable {variable_id} was not found") - return MetadataDetailResult( - policyengine_version=selected.policyengine_version, - item=_variable(rows[0]), - ) diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py index d9922ac06..97215a526 100644 --- a/policyengine_api/data/v2/database.py +++ b/policyengine_api/data/v2/database.py @@ -20,7 +20,7 @@ DATABASE_POOL_RECYCLE_SECONDS = 1800 DATABASE_POOL_SIZE = 5 DATABASE_POOL_MAX_OVERFLOW = 5 -DATABASE_POOL_TIMEOUT_SECONDS = 30 +DATABASE_TIMEOUT_SECONDS = 5 _state_lock = Lock() _engine: Engine | None = None @@ -55,7 +55,11 @@ def build_v2_engine(settings: V2DatabaseSettings) -> Engine: pool_recycle=DATABASE_POOL_RECYCLE_SECONDS, pool_size=DATABASE_POOL_SIZE, max_overflow=DATABASE_POOL_MAX_OVERFLOW, - pool_timeout=DATABASE_POOL_TIMEOUT_SECONDS, + pool_timeout=DATABASE_TIMEOUT_SECONDS, + connect_args={ + "connect_timeout": DATABASE_TIMEOUT_SECONDS, + "options": (f"-c statement_timeout={DATABASE_TIMEOUT_SECONDS * 1_000}"), + }, ) diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 491207ef4..4459e8c11 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -51,6 +51,11 @@ UserReportAssociation, UserSimulationAssociation, ) +from policyengine_api.data.v2.models.policy_mappings import ( # noqa: E402 + LegacyPolicyMapping, + LegacyUserMapping, + LegacyUserPolicyMapping, +) from policyengine_api.data.v2.models.reports import ( # noqa: E402 AggregateOutput, AggregateType, @@ -94,6 +99,9 @@ "Inequality", "IntraDecileImpact", "LocalAuthorityImpact", + "LegacyPolicyMapping", + "LegacyUserMapping", + "LegacyUserPolicyMapping", "OutputStatus", "Parameter", "ParameterNode", diff --git a/policyengine_api/data/v2/models/associations.py b/policyengine_api/data/v2/models/associations.py index e15dd60e6..38ba0ce77 100644 --- a/policyengine_api/data/v2/models/associations.py +++ b/policyengine_api/data/v2/models/associations.py @@ -1,7 +1,7 @@ """Canonical SQLModel tables linking v2 users to their domain records.""" from datetime import datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from uuid import UUID import sqlalchemy as sa @@ -14,6 +14,9 @@ from policyengine_api.data.v2.models.users import User if TYPE_CHECKING: + from policyengine_api.data.v2.models.policy_mappings import ( + LegacyUserPolicyMapping, + ) from policyengine_api.data.v2.models.reports import Report @@ -48,9 +51,31 @@ class UserPolicy(TimestampedModel, table=True): __tablename__ = "user_policies" __table_args__ = ( sa.UniqueConstraint( + "id", + "country_id", + name="uq_user_policies_id_country", + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name="ck_user_policies_country", + ), + sa.ForeignKeyConstraint( + ["policy_id", "country_id"], + ["policies.id", "policies.country_id"], + name="fk_user_policies_policy_country", + ondelete="RESTRICT", + ), + sa.Index( + "ix_user_policies_country_user_created_id", + "country_id", "user_id", + "created_at", + "id", + ), + sa.Index( + "ix_user_policies_country_policy", + "country_id", "policy_id", - name="uq_user_policies_user_policy", ), ) @@ -59,16 +84,17 @@ class UserPolicy(TimestampedModel, table=True): ondelete="CASCADE", index=True, ) - policy_id: UUID = Field( - foreign_key="policies.id", - ondelete="CASCADE", - index=True, - ) - country: str = Field(max_length=16) - label: str | None = Field(default=None, max_length=255) + policy_id: UUID = Field(index=True) + country_id: str = Field(max_length=2) + name: str | None = Field(default=None, max_length=255) + description: str | None = Field(default=None, sa_type=sa.Text) user: User = Relationship(back_populates="policy_associations") policy: Policy = Relationship(back_populates="user_associations") + legacy_mapping: Optional["LegacyUserPolicyMapping"] = Relationship( + back_populates="association", + cascade_delete=True, + ) class UserSimulationAssociation(TimestampedModel, table=True): diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index ab452d5e2..d1a797187 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -87,6 +87,7 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): ) datasets: list["Dataset"] = Relationship(back_populates="tax_benefit_model_version") regions: list["Region"] = Relationship(back_populates="tax_benefit_model_version") + policies: list["Policy"] = Relationship(back_populates="tax_benefit_model_version") class Region(TimestampedModel, table=True): @@ -314,6 +315,16 @@ class ParameterValue(IdentifiedModel, table=True): "policy_id IS NULL OR dynamic_id IS NULL", name="ck_parameter_values_single_owner", ), + sa.CheckConstraint( + "end_date IS NULL OR end_date >= start_date", + name="ck_parameter_values_effective_period", + ), + sa.UniqueConstraint( + "policy_id", + "parameter_id", + "start_date", + name="uq_parameter_values_policy_parameter_start_date", + ), sa.Index( "ix_parameter_values_parameter_period", "parameter_id", @@ -337,7 +348,9 @@ class ParameterValue(IdentifiedModel, table=True): foreign_key="parameters.id", ondelete="CASCADE", ) - value_json: Any = Field(sa_type=sa.JSON) + value_json: Any = Field( + sa_type=sa.JSON().with_variant(sa.dialects.postgresql.JSONB(), "postgresql") + ) start_date: datetime = Field(sa_type=sa.DateTime(timezone=True)) end_date: datetime | None = Field( default=None, diff --git a/policyengine_api/data/v2/models/policies.py b/policyengine_api/data/v2/models/policies.py index ba632a240..9b9957b30 100644 --- a/policyengine_api/data/v2/models/policies.py +++ b/policyengine_api/data/v2/models/policies.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING from uuid import UUID +import sqlalchemy as sa from sqlmodel import Field, Relationship from policyengine_api.data.v2.models.base import TimestampedModel @@ -11,23 +12,70 @@ if TYPE_CHECKING: from policyengine_api.data.v2.models.associations import UserPolicy from policyengine_api.data.v2.models.households import HouseholdJob - from policyengine_api.data.v2.models.metadata import ParameterValue + from policyengine_api.data.v2.models.metadata import ( + ParameterValue, + TaxBenefitModelVersion, + ) + from policyengine_api.data.v2.models.policy_mappings import LegacyPolicyMapping from policyengine_api.data.v2.models.reports import Report from policyengine_api.data.v2.models.simulations import Simulation class Policy(TimestampedModel, table=True): __tablename__ = "policies" + __table_args__ = ( + sa.UniqueConstraint( + "id", + "country_id", + name="uq_policies_id_country", + ), + sa.UniqueConstraint( + "canonicalization_version", + "content_hash", + name="uq_policies_canonicalization_content_hash", + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name="ck_policies_country", + ), + sa.CheckConstraint( + "canonicalization_version > 0", + name="ck_policies_canonicalization_version", + ), + sa.CheckConstraint( + "length(content_hash) = 64", + name="ck_policies_content_hash_length", + ), + sa.Index( + "ix_policies_country_model", + "country_id", + "tax_benefit_model_id", + ), + sa.Index( + "ix_policies_country_model_version", + "country_id", + "tax_benefit_model_version_id", + ), + ) - name: str = Field(max_length=255) - description: str | None = None + country_id: str = Field(max_length=2) tax_benefit_model_id: UUID = Field( foreign_key="tax_benefit_models.id", ondelete="RESTRICT", index=True, ) + tax_benefit_model_version_id: UUID = Field( + foreign_key="tax_benefit_model_versions.id", + ondelete="RESTRICT", + index=True, + ) + canonicalization_version: int + content_hash: str = Field(max_length=64) tax_benefit_model: TaxBenefitModel = Relationship(back_populates="policies") + tax_benefit_model_version: "TaxBenefitModelVersion" = Relationship( + back_populates="policies" + ) parameter_values: list["ParameterValue"] = Relationship( back_populates="policy", cascade_delete=True, @@ -37,8 +85,8 @@ class Policy(TimestampedModel, table=True): reports: list["Report"] = Relationship(back_populates="policy") user_associations: list["UserPolicy"] = Relationship( back_populates="policy", - cascade_delete=True, ) + legacy_mappings: list["LegacyPolicyMapping"] = Relationship(back_populates="policy") class Dynamic(TimestampedModel, table=True): diff --git a/policyengine_api/data/v2/models/policy_mappings.py b/policyengine_api/data/v2/models/policy_mappings.py new file mode 100644 index 000000000..7cdbc91b9 --- /dev/null +++ b/policyengine_api/data/v2/models/policy_mappings.py @@ -0,0 +1,128 @@ +"""Durable source-identity mappings for immediate v1 policy mirroring.""" + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Field, Relationship, SQLModel + +from policyengine_api.data.v2.models.base import TimestampedModel, utc_now + +if TYPE_CHECKING: + from policyengine_api.data.v2.models.associations import UserPolicy + from policyengine_api.data.v2.models.policies import Policy + + +class LegacyUserMapping(SQLModel, table=True): + """Map one exact v1 user identifier to one v2 user UUID.""" + + __tablename__ = "legacy_user_mappings" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + name="uq_legacy_user_mappings_user_id", + ), + ) + + legacy_user_id: str = Field(primary_key=True, max_length=255) + user_id: UUID = Field( + foreign_key="users.id", + ondelete="RESTRICT", + index=True, + ) + created_at: datetime = Field( + default_factory=utc_now, + sa_type=sa.DateTime(timezone=True), + sa_column_kwargs={"server_default": sa.func.now()}, + ) + + +class LegacyPolicyMapping(TimestampedModel, table=True): + """Map one country-scoped v1 policy ID to deduplicated v2 content.""" + + __tablename__ = "legacy_policy_mappings" + __table_args__ = ( + sa.UniqueConstraint( + "country_id", + "legacy_policy_id", + name="uq_legacy_policy_mappings_country_legacy", + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name="ck_legacy_policy_mappings_country", + ), + sa.ForeignKeyConstraint( + ["policy_id", "country_id"], + ["policies.id", "policies.country_id"], + name="fk_legacy_policy_mappings_policy_country", + ondelete="RESTRICT", + ), + sa.Index( + "ix_legacy_policy_mappings_policy", + "policy_id", + ), + ) + + country_id: str = Field(max_length=2) + legacy_policy_id: int = Field(sa_type=sa.BigInteger) + policy_id: UUID + source_policy_hash: str = Field(max_length=255) + + policy: "Policy" = Relationship(back_populates="legacy_mappings") + + +class LegacyUserPolicyMapping(TimestampedModel, table=True): + """Map one country-scoped v1 saved policy to one v2 association.""" + + __tablename__ = "legacy_user_policy_mappings" + __table_args__ = ( + sa.UniqueConstraint( + "country_id", + "legacy_user_policy_id", + name="uq_legacy_user_policy_mappings_country_legacy", + ), + sa.UniqueConstraint( + "user_policy_id", + name="uq_legacy_user_policy_mappings_association", + ), + sa.CheckConstraint( + "country_id IN ('us', 'uk')", + name="ck_legacy_user_policy_mappings_country", + ), + sa.CheckConstraint( + "fingerprint_version > 0", + name="ck_legacy_user_policy_mappings_fingerprint_version", + ), + sa.CheckConstraint( + "last_applied_source_revision >= 0", + name="ck_legacy_user_policy_mappings_source_revision", + ), + sa.CheckConstraint( + "length(fingerprint_sha256) = 64", + name="ck_legacy_user_policy_mappings_fingerprint_length", + ), + sa.ForeignKeyConstraint( + ["user_policy_id", "country_id"], + ["user_policies.id", "user_policies.country_id"], + name="fk_legacy_user_policy_mappings_association_country", + ondelete="CASCADE", + ), + sa.Index( + "ix_legacy_user_policy_mappings_association", + "user_policy_id", + ), + ) + + country_id: str = Field(max_length=2) + legacy_user_policy_id: int = Field(sa_type=sa.BigInteger) + user_policy_id: UUID + last_applied_source_revision: int = Field( + default=0, + sa_type=sa.BigInteger, + sa_column_kwargs={"server_default": "0"}, + ) + fingerprint_version: int + fingerprint_sha256: str = Field(max_length=64) + + association: "UserPolicy" = Relationship(back_populates="legacy_mapping") diff --git a/policyengine_api/data/v2/models/users.py b/policyengine_api/data/v2/models/users.py index fc7582a5a..87db70d6b 100644 --- a/policyengine_api/data/v2/models/users.py +++ b/policyengine_api/data/v2/models/users.py @@ -27,9 +27,9 @@ class User(IdentifiedModel, table=True): ), ) - first_name: str = Field(max_length=255) - last_name: str = Field(max_length=255) - email: str = Field(max_length=320, index=True) + first_name: str | None = Field(default=None, max_length=255) + last_name: str | None = Field(default=None, max_length=255) + email: str | None = Field(default=None, max_length=320, index=True) primary_country: str = Field(max_length=2) reports: list["Report"] = Relationship(back_populates="user") diff --git a/policyengine_api/data/v2/policy_migration_qualification.py b/policyengine_api/data/v2/policy_migration_qualification.py new file mode 100644 index 000000000..e2224844e --- /dev/null +++ b/policyengine_api/data/v2/policy_migration_qualification.py @@ -0,0 +1,255 @@ +"""Read-only qualification for dormant v2 policy data.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping +from dataclasses import dataclass +import json +import sys +from typing import Protocol + +from alembic.config import Config +from alembic.migration import MigrationContext +from alembic.script import ScriptDirectory +from sqlalchemy import Connection, Engine, func, inspect, select +from sqlalchemy.exc import NoInspectionAvailable +from sqlalchemy.pool import NullPool +from sqlmodel import create_engine + +from policyengine_api.data.v2.models import ParameterValue, Policy, UserPolicy +from policyengine_api.data.v2.settings import ( + V2ConfigurationError, + V2DatabaseSettings, + load_v2_migration_database_settings, +) + + +POLICY_MIGRATION_REVISION = "711ec2f0a5a5" +V2_ALEMBIC_CONFIG = "alembic-v2.ini" + + +class ScalarExecutor(Protocol): + """Minimal database interface required by the row-count queries.""" + + def scalar(self, statement: object) -> object | None: + """Return the first column from the first result row.""" + + +@dataclass(frozen=True) +class PolicyDataCounts: + """Counts of predecessor policy data that the migration would replace.""" + + policies: int + policy_parameter_values: int + user_policies: int + + @property + def total(self) -> int: + return self.policies + self.policy_parameter_values + self.user_policies + + def as_dict(self) -> dict[str, int]: + return { + "policies": self.policies, + "policy_parameter_values": self.policy_parameter_values, + "user_policies": self.user_policies, + } + + +@dataclass(frozen=True) +class PolicyMigrationQualification: + """Non-secret evidence that one configured Supabase target is empty.""" + + environment: str + project_ref: str + counts: PolicyDataCounts + required: bool = True + + def as_dict(self) -> dict[str, object]: + return { + "outcome": "ok", + "qualification": "performed" if self.required else "not-required", + "environment": self.environment, + "project_ref": self.project_ref, + "counts": self.counts.as_dict(), + } + + +class RetainedPolicyDataError(RuntimeError): + """Raised when a target contains policy data requiring preservation.""" + + def __init__(self, counts: PolicyDataCounts) -> None: + self.counts = counts + super().__init__( + "the configured Supabase target contains retained v2 policy data " + f"(policies={counts.policies}, " + f"policy_parameter_values={counts.policy_parameter_values}, " + f"user_policies={counts.user_policies}); migration stopped without " + "modifying data. Use an empty target or obtain a reviewed data-" + "preservation plan before retrying" + ) + + +def read_policy_data_counts( + executor: ScalarExecutor, + existing_tables: Collection[str] | None = None, +) -> PolicyDataCounts: + """Count only policy-owned rows; canonical catalog values are excluded.""" + + def table_exists(name: str) -> bool: + return existing_tables is None or name in existing_tables + + return PolicyDataCounts( + policies=( + int(executor.scalar(select(func.count()).select_from(Policy)) or 0) + if table_exists("policies") + else 0 + ), + policy_parameter_values=int( + executor.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id.is_not(None)) + ) + or 0 + ) + if table_exists("parameter_values") + else 0, + user_policies=( + int(executor.scalar(select(func.count()).select_from(UserPolicy)) or 0) + if table_exists("user_policies") + else 0 + ), + ) + + +def require_no_retained_policy_data(counts: PolicyDataCounts) -> None: + """Stop the migration when any predecessor policy row requires a decision.""" + + if counts.total: + raise RetainedPolicyDataError(counts) + + +def build_qualification_engine(settings: V2DatabaseSettings) -> Engine: + """Build an isolated connection pool for one qualification attempt.""" + + return create_engine(settings.connection.url, poolclass=NullPool) + + +def _read_and_require_counts(connection: Connection) -> PolicyDataCounts: + try: + existing_tables = inspect(connection).get_table_names(schema="public") + except NoInspectionAvailable: + existing_tables = None + counts = read_policy_data_counts(connection, existing_tables) + require_no_retained_policy_data(counts) + return counts + + +def _qualify_connection(connection: Connection) -> PolicyDataCounts: + transaction = connection.begin() + try: + connection.exec_driver_sql("SET TRANSACTION READ ONLY") + return _read_and_require_counts(connection) + finally: + transaction.rollback() + + +def policy_migration_is_pending(connection: Connection) -> bool: + """Return whether the Phase 10 policy revision is in the upgrade path.""" + + current_heads = MigrationContext.configure(connection).get_current_heads() + revisions = ScriptDirectory.from_config( + Config(V2_ALEMBIC_CONFIG) + ).iterate_revisions("heads", current_heads) + return POLICY_MIGRATION_REVISION in {migration.revision for migration in revisions} + + +def qualify_policy_migration_target( + environ: Mapping[str, str] | None = None, + *, + engine_builder: Callable[[V2DatabaseSettings], Engine] = ( + build_qualification_engine + ), +) -> PolicyMigrationQualification: + """Qualify the configured Supabase target without changing it.""" + + settings = load_v2_migration_database_settings(environ) + engine = engine_builder(settings) + try: + with engine.connect() as connection: + counts = _qualify_connection(connection) + finally: + engine.dispose() + return PolicyMigrationQualification( + environment=settings.target.environment, + project_ref=settings.target.project_ref, + counts=counts, + ) + + +def qualify_policy_migration_if_pending( + environ: Mapping[str, str] | None = None, + *, + engine_builder: Callable[[V2DatabaseSettings], Engine] = ( + build_qualification_engine + ), + pending_checker: Callable[[Connection], bool] = policy_migration_is_pending, +) -> PolicyMigrationQualification: + """Qualify predecessor rows only while the Phase 10 revision is pending.""" + + settings = load_v2_migration_database_settings(environ) + engine = engine_builder(settings) + try: + with engine.connect() as connection: + transaction = connection.begin() + try: + connection.exec_driver_sql("SET TRANSACTION READ ONLY") + if not pending_checker(connection): + return PolicyMigrationQualification( + environment=settings.target.environment, + project_ref=settings.target.project_ref, + counts=PolicyDataCounts(0, 0, 0), + required=False, + ) + counts = _read_and_require_counts(connection) + finally: + transaction.rollback() + finally: + engine.dispose() + return PolicyMigrationQualification( + environment=settings.target.environment, + project_ref=settings.target.project_ref, + counts=counts, + ) + + +def _error_payload(error: Exception) -> dict[str, object]: + safe_errors = (V2ConfigurationError, RetainedPolicyDataError) + message = ( + str(error) + if isinstance(error, safe_errors) + else "v2 policy migration qualification failed unexpectedly" + ) + return { + "outcome": "error", + "error": { + "type": type(error).__name__, + "message": message, + }, + } + + +def main() -> int: + """Run qualification and return a shell-compatible status.""" + + try: + evidence = qualify_policy_migration_if_pending() + except Exception as error: # noqa: BLE001 - command must emit safe evidence + print(json.dumps(_error_payload(error), sort_keys=True), file=sys.stderr) + return 1 + print(json.dumps(evidence.as_dict(), sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/policyengine_api/data/v2/settings.py b/policyengine_api/data/v2/settings.py index a75b83def..06920ac9c 100644 --- a/policyengine_api/data/v2/settings.py +++ b/policyengine_api/data/v2/settings.py @@ -32,6 +32,7 @@ ENVIRONMENT_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,31}$") SECRET_RESOURCE_PATTERN = re.compile(r"^projects/[^/]+/secrets/[^/]+/versions/[^/]+$") SUPABASE_DATABASE_NAME = "postgres" +SUPABASE_TRANSACTION_POOLER_PORT = 6543 class V2ConfigurationError(RuntimeError): @@ -249,11 +250,18 @@ def load_v2_runtime_database_settings( values, secret_loader=secret_loader or _load_secret_from_secret_manager, ) - return _database_settings( + settings = _database_settings( raw_url, setting_name=V2_RUNTIME_DATABASE_URL, environ=values, ) + if settings.connection.url.port == SUPABASE_TRANSACTION_POOLER_PORT: + raise V2ConfigurationError( + f"{V2_RUNTIME_DATABASE_URL} must use a direct connection or " + "Supavisor session mode on port 5432 so the runtime statement " + "timeout is applied" + ) + return settings def load_v2_migration_database_settings( diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 4b3166689..9352a7074 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -4,12 +4,42 @@ from collections.abc import Callable from dataclasses import dataclass +from datetime import datetime from functools import lru_cache from importlib import metadata as importlib_metadata -from typing import Protocol +from typing import TYPE_CHECKING, Protocol +from uuid import UUID from policyengine_api.json_types import JSONObject +if TYPE_CHECKING: + from policyengine_api.services.v2.metadata.types import ( + MetadataCanonicalParameterValue, + MetadataDataset, + MetadataDetailResult, + MetadataEconomyOptionsResult, + MetadataModel, + MetadataModelSelectionResult, + MetadataModelVersionDetail, + MetadataPageResult, + MetadataParameterChild, + MetadataParameterSummary, + MetadataRegion, + MetadataVariable, + ) + from policyengine_api.services.v2.policies.types import ( + NativePolicyCreation, + NativePolicyCreationInput, + PolicyPage, + PolicyRead, + ) + from policyengine_api.services.v2.user_policies.types import ( + UserPolicyCreationInput, + UserPolicyPage, + UserPolicyRead, + UserPolicyUpdateInput, + ) + class MetadataReader(Protocol): """Read the already-loaded metadata document for a country.""" @@ -22,6 +52,214 @@ class V2MetadataResourceReader(Protocol): def close(self) -> None: ... + def list_models( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataModel]": ... + + def get_model( + self, + country_id: str, + model_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataModel]": ... + + def get_model_by_country( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> "MetadataModelSelectionResult": ... + + def list_model_versions( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataModelVersionDetail]": ... + + def get_model_version( + self, + country_id: str, + version_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataModelVersionDetail]": ... + + def list_variables( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + search: str | None = None, + ) -> "MetadataPageResult[MetadataVariable]": ... + + def get_variable( + self, + country_id: str, + variable_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataVariable]": ... + + def list_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + search: str | None = None, + ) -> "MetadataPageResult[MetadataParameterSummary]": ... + + def get_parameter( + self, + country_id: str, + parameter_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataParameterSummary]": ... + + def list_parameter_children( + self, + country_id: str, + policyengine_version: str | None = None, + *, + parent_path: str = "", + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataParameterChild]": ... + + def list_parameter_values( + self, + country_id: str, + policyengine_version: str | None = None, + *, + parameter_id: UUID | None = None, + current: bool = False, + offset: int = 0, + limit: int = 100, + now: datetime | None = None, + ) -> "MetadataPageResult[MetadataCanonicalParameterValue]": ... + + def get_parameter_value( + self, + country_id: str, + value_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataCanonicalParameterValue]": ... + + def list_datasets( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataDataset]": ... + + def get_dataset( + self, + country_id: str, + dataset_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataDataset]": ... + + def list_regions( + self, + country_id: str, + policyengine_version: str | None = None, + *, + region_type: str | None = None, + offset: int = 0, + limit: int = 100, + ) -> "MetadataPageResult[MetadataRegion]": ... + + def get_region( + self, + country_id: str, + region_id: UUID, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataRegion]": ... + + def get_region_by_code( + self, + country_id: str, + region_code: str, + policyengine_version: str | None = None, + ) -> "MetadataDetailResult[MetadataRegion]": ... + + def get_economy_options( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> "MetadataEconomyOptionsResult": ... + + +class V2PolicyResourceService(Protocol): + """Route-independent native policy operations for one request.""" + + def create_policy( + self, + command: "NativePolicyCreationInput", + ) -> "NativePolicyCreation": ... + + def get_policy(self, *, country_id: str, policy_id: UUID) -> "PolicyRead": ... + + def list_policies( + self, + *, + country_id: str, + tax_benefit_model_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> "PolicyPage": ... + + +class V2UserPolicyResourceService(Protocol): + """Route-independent native association operations for one request.""" + + def create_user_policy( + self, + association_input: "UserPolicyCreationInput", + ) -> "UserPolicyRead": ... + + def get_user_policy( + self, + *, + country_id: str, + association_id: UUID, + ) -> "UserPolicyRead": ... + + def list_user_policies( + self, + *, + country_id: str, + user_id: UUID, + policy_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> "UserPolicyPage": ... + + def patch_user_policy( + self, + *, + country_id: str, + association_id: UUID, + association_input: "UserPolicyUpdateInput", + ) -> "UserPolicyRead": ... + + def delete_user_policy( + self, + *, + country_id: str, + association_id: UUID, + ) -> None: ... + class SimulationGatewayProbe(Protocol): """Minimal simulation-entrypoint health-check interface.""" @@ -59,15 +297,43 @@ def _running_policyengine_version() -> str: def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: - from policyengine_api.data.v2.catalog.query import V2MetadataQueryService from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.services.v2.metadata.database_session import ( + MetadataDatabaseSession, + ) + from policyengine_api.services.v2.metadata.services import V2MetadataService + + return V2MetadataService( + MetadataDatabaseSession(get_v2_session_factory()()), + running_policyengine_version=_running_policyengine_version(), + ) + + +def _default_v2_policy_service_factory() -> V2PolicyResourceService: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, + ) + from policyengine_api.services.v2.policies.services import V2PolicyService - return V2MetadataQueryService( - get_v2_session_factory()(), + return V2PolicyService( + PolicyDatabaseSession(get_v2_session_factory()), running_policyengine_version=_running_policyengine_version(), ) +def _default_v2_user_policy_service_factory() -> V2UserPolicyResourceService: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, + ) + from policyengine_api.services.v2.user_policies.services import ( + V2UserPolicyService, + ) + + return V2UserPolicyService(UserPolicyDatabaseSession(get_v2_session_factory())) + + @dataclass(frozen=True) class NativeRouteDependencies: """Runtime collaborators for native read routes.""" @@ -77,6 +343,10 @@ class NativeRouteDependencies: metadata_reader_factory: Callable[[], MetadataReader] specification_provider: Callable[[], JSONObject] v2_metadata_reader_factory: Callable[[], V2MetadataResourceReader] | None = None + v2_policy_service_factory: Callable[[], V2PolicyResourceService] | None = None + v2_user_policy_service_factory: Callable[[], V2UserPolicyResourceService] | None = ( + None + ) @classmethod def defaults(cls) -> "NativeRouteDependencies": @@ -87,4 +357,6 @@ def defaults(cls) -> "NativeRouteDependencies": metadata_reader_factory=_default_metadata_reader_factory, specification_provider=_default_specification_provider, v2_metadata_reader_factory=_default_v2_metadata_reader_factory, + v2_policy_service_factory=_default_v2_policy_service_factory, + v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), ) diff --git a/policyengine_api/fastapi_routes/query_parameters.py b/policyengine_api/fastapi_routes/query_parameters.py new file mode 100644 index 000000000..98670d410 --- /dev/null +++ b/policyengine_api/fastapi_routes/query_parameters.py @@ -0,0 +1,66 @@ +"""FastAPI dependency adapter for canonical query-parameter models.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from inspect import Parameter, Signature +from typing import Annotated, TypeVar + +from fastapi import Query, Request +from fastapi.exceptions import RequestValidationError + +from policyengine_api.query_parameters import ( + DuplicateScalarQueryParameterError, + StrictQueryParameters, + validate_scalar_query_multiplicity, +) + + +QueryParametersT = TypeVar("QueryParametersT", bound=StrictQueryParameters) + + +def _duplicate_error(error: DuplicateScalarQueryParameterError) -> dict[str, object]: + return { + "type": "value_error", + "loc": ("query", error.parameter_name), + "msg": "Input should occur only once for a scalar query parameter", + "input": None, + "ctx": {"error": error}, + } + + +def query_dependency( + model_type: type[QueryParametersT], +) -> Callable[..., Awaitable[QueryParametersT]]: + """Build a typed dependency with runtime and OpenAPI query metadata.""" + + async def dependency( + request: Request, + query: QueryParametersT, + ) -> QueryParametersT: + try: + validate_scalar_query_multiplicity( + model_type, + request.query_params.multi_items(), + ) + except DuplicateScalarQueryParameterError as error: + raise RequestValidationError([_duplicate_error(error)]) from error + return query + + dependency.__name__ = f"parse_{model_type.__name__}" + dependency.__signature__ = Signature( # type: ignore[attr-defined] + parameters=( + Parameter( + "request", + kind=Parameter.POSITIONAL_OR_KEYWORD, + annotation=Request, + ), + Parameter( + "query", + kind=Parameter.POSITIONAL_OR_KEYWORD, + annotation=Annotated[model_type, Query()], + ), + ), + return_annotation=model_type, + ) + return dependency diff --git a/policyengine_api/fastapi_routes/v2/__init__.py b/policyengine_api/fastapi_routes/v2/__init__.py new file mode 100644 index 000000000..c4f3cad04 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/__init__.py @@ -0,0 +1 @@ +"""Native API v2 HTTP route modules.""" diff --git a/policyengine_api/fastapi_routes/v2/errors.py b/policyengine_api/fastapi_routes/v2/errors.py new file mode 100644 index 000000000..3f69cb9f6 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/errors.py @@ -0,0 +1,29 @@ +"""Shared error response contract for all native API v2 routes.""" + +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, StringConstraints +from starlette.responses import JSONResponse + + +class V2RequestTooLargeError(ValueError): + """Raised when an API v2 request exceeds its documented byte limit.""" + + +class V2ErrorResponse(BaseModel): + """Strict error envelope shared by every API v2 resource.""" + + model_config = ConfigDict(extra="forbid") + + status: Literal["error"] = "error" + message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] + + +def v2_error_response(status_code: int, message: str) -> JSONResponse: + """Serialize one API v2 error without resource-path dispatch.""" + + error = V2ErrorResponse(message=message) + return JSONResponse( + status_code=status_code, + content=error.model_dump(mode="json"), + ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/__init__.py b/policyengine_api/fastapi_routes/v2/metadata/__init__.py new file mode 100644 index 000000000..7cff9ae11 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/__init__.py @@ -0,0 +1 @@ +"""Native API v2 metadata HTTP route modules.""" diff --git a/policyengine_api/fastapi_routes/v2_metadata_common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py similarity index 64% rename from policyengine_api/fastapi_routes/v2_metadata_common.py rename to policyengine_api/fastapi_routes/v2/metadata/common.py index de54e75c4..fefc8b0a9 100644 --- a/policyengine_api/fastapi_routes/v2_metadata_common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -1,69 +1,69 @@ -"""Shared response handling for dormant v2 metadata resources.""" +"""Execute API v2 metadata reads and translate failures to HTTP responses.""" from __future__ import annotations from collections.abc import Callable -from typing import TypeVar +from typing import Any, TypeVar from pydantic import BaseModel from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.query import ( - InvalidMetadataPageError, +from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, - MetadataResourceNotFoundError, UnsupportedPreviewCountryError, ) -from policyengine_api.data.v2.catalog.schemas import MetadataErrorResponse +from policyengine_api.services.v2.metadata.validators import ( + InvalidMetadataPageError, + MetadataResourceNotFoundError, +) +from policyengine_api.fastapi_routes.v2.errors import ( + V2ErrorResponse, + v2_error_response, +) from policyengine_api.data.v2.settings import V2ConfigurationError -from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2MetadataResourceReader, +) -ERROR_RESPONSES = { +ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The resource request or PolicyEngine.py version is invalid.", }, 404: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The requested catalog resource is absent.", }, 405: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The dormant v2 metadata resources support GET only.", }, 422: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The request parameters do not match the resource schema.", }, 500: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The resource query failed internally.", }, 503: { - "model": MetadataErrorResponse, + "model": V2ErrorResponse, "description": "The initialized v2 catalog is unavailable.", }, } -def error_response(status_code: int, message: str) -> JSONResponse: - error = MetadataErrorResponse(message=message) - return JSONResponse( - status_code=status_code, - content=error.model_dump(mode="json"), - ) - - ResponseT = TypeVar("ResponseT", bound=BaseModel) def read_resource( dependencies: NativeRouteDependencies, response_type: type[ResponseT], - operation: Callable[[object], object], + operation: Callable[[V2MetadataResourceReader], object], ) -> ResponseT | JSONResponse: reader = None try: @@ -77,18 +77,18 @@ def read_resource( reader = factory() return response_type(result=operation(reader)) except (InvalidMetadataPageError, InvalidPolicyEngineVersionError) as error: - return error_response(400, str(error)) + return v2_error_response(400, str(error)) except UnsupportedPreviewCountryError as error: - return error_response(400, f"Unsupported country: {error}") + return v2_error_response(400, f"Unsupported country: {error}") except ( MetadataCatalogVersionNotFoundError, MetadataResourceNotFoundError, ) as error: - return error_response(404, str(error)) + return v2_error_response(404, str(error)) except (V2ConfigurationError, MetadataCatalogUnavailableError): - return error_response(503, "V2 metadata catalog is unavailable") + return v2_error_response(503, "V2 metadata catalog is unavailable") except Exception: # noqa: BLE001 - preview must return typed errors - return error_response(500, "V2 metadata query failed") + return v2_error_response(500, "V2 metadata query failed") finally: if reader is not None: try: diff --git a/policyengine_api/fastapi_routes/v2_metadata_geography.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py similarity index 94% rename from policyengine_api/fastapi_routes/v2_metadata_geography.py rename to policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index fc366b0f6..e3da1f81e 100644 --- a/policyengine_api/fastapi_routes/v2_metadata_geography.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -1,4 +1,4 @@ -"""Dataset, region, and economy-option preview routes.""" +"""API v2 dataset, region, and economy-option routes.""" from __future__ import annotations @@ -8,16 +8,16 @@ from fastapi import APIRouter, Query from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.services.v2.metadata.types import MetadataRegionType +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataDatasetDetailResponse, MetadataDatasetPageResponse, MetadataEconomyOptionsResponse, MetadataRegionDetailResponse, MetadataRegionPageResponse, - MetadataRegionType, ) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -from policyengine_api.fastapi_routes.v2_metadata_common import ( +from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, read_resource, ) diff --git a/policyengine_api/fastapi_routes/v2_metadata_models.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py similarity index 96% rename from policyengine_api/fastapi_routes/v2_metadata_models.py rename to policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 8a1147f62..01693f512 100644 --- a/policyengine_api/fastapi_routes/v2_metadata_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -1,4 +1,4 @@ -"""Tax-benefit model, version, and variable preview routes.""" +"""API v2 tax-benefit model, version, and variable routes.""" from __future__ import annotations @@ -8,7 +8,7 @@ from fastapi import APIRouter, Query from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataModelDetailResponse, MetadataModelPageResponse, MetadataModelSelectionResponse, @@ -18,7 +18,7 @@ MetadataVariablePageResponse, ) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -from policyengine_api.fastapi_routes.v2_metadata_common import ( +from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, read_resource, ) diff --git a/policyengine_api/fastapi_routes/v2_metadata_parameters.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py similarity index 95% rename from policyengine_api/fastapi_routes/v2_metadata_parameters.py rename to policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index b4f17cb20..714c3e28d 100644 --- a/policyengine_api/fastapi_routes/v2_metadata_parameters.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -1,4 +1,4 @@ -"""Parameter and canonical parameter-value preview routes.""" +"""API v2 parameter and canonical parameter-value routes.""" from __future__ import annotations @@ -8,7 +8,7 @@ from fastapi import APIRouter, Query from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataParameterChildPageResponse, MetadataParameterDetailResponse, MetadataParameterPageResponse, @@ -16,7 +16,7 @@ MetadataParameterValuePageResponse, ) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -from policyengine_api.fastapi_routes.v2_metadata_common import ( +from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, read_resource, ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py new file mode 100644 index 000000000..8d6d965a9 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -0,0 +1,134 @@ +"""Strict HTTP response models for API v2 metadata resources.""" + +from __future__ import annotations + +from typing import Generic, Literal, TypeVar + +from policyengine_api.services.v2.metadata.types import ( + MetadataCanonicalParameterValue, + MetadataDataset, + MetadataDetailResult, + MetadataEconomyOptionsResult, + MetadataModel, + MetadataModelSelectionResult, + MetadataModelVersionDetail, + MetadataPageResult, + MetadataParameterChild, + MetadataParameterSummary, + MetadataRegion, + MetadataVariable, + StrictResponseModel, +) + + +ResourceT = TypeVar("ResourceT") + + +class MetadataResourceSuccessResponse(StrictResponseModel, Generic[ResourceT]): + status: Literal["ok"] = "ok" + message: None = None + result: ResourceT + + +class MetadataModelPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataModel]] +): + pass + + +class MetadataModelDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataModel]] +): + pass + + +class MetadataModelSelectionResponse( + MetadataResourceSuccessResponse[MetadataModelSelectionResult] +): + pass + + +class MetadataModelVersionPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataModelVersionDetail]] +): + pass + + +class MetadataModelVersionDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataModelVersionDetail]] +): + pass + + +class MetadataVariablePageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataVariable]] +): + pass + + +class MetadataVariableDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataVariable]] +): + pass + + +class MetadataParameterPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataParameterSummary]] +): + pass + + +class MetadataParameterDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataParameterSummary]] +): + pass + + +class MetadataParameterChildPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataParameterChild]] +): + pass + + +class MetadataParameterValuePageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataCanonicalParameterValue]] +): + pass + + +class MetadataParameterValueDetailResponse( + MetadataResourceSuccessResponse[ + MetadataDetailResult[MetadataCanonicalParameterValue] + ] +): + pass + + +class MetadataDatasetPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataDataset]] +): + pass + + +class MetadataDatasetDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataDataset]] +): + pass + + +class MetadataRegionPageResponse( + MetadataResourceSuccessResponse[MetadataPageResult[MetadataRegion]] +): + pass + + +class MetadataRegionDetailResponse( + MetadataResourceSuccessResponse[MetadataDetailResult[MetadataRegion]] +): + pass + + +class MetadataEconomyOptionsResponse( + MetadataResourceSuccessResponse[MetadataEconomyOptionsResult] +): + pass diff --git a/policyengine_api/fastapi_routes/v2/policies/__init__.py b/policyengine_api/fastapi_routes/v2/policies/__init__.py new file mode 100644 index 000000000..73b4861b0 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/policies/__init__.py @@ -0,0 +1 @@ +"""FastAPI adapters for immutable v2 policies.""" diff --git a/policyengine_api/fastapi_routes/v2/policies/request_models.py b/policyengine_api/fastapi_routes/v2/policies/request_models.py new file mode 100644 index 000000000..e3180b94f --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/policies/request_models.py @@ -0,0 +1,10 @@ +"""Strict HTTP request models for the native v2 policy API.""" + +from policyengine_api.services.v2.policies.types import PolicyCreationInput + + +MAXIMUM_POLICY_REQUEST_BYTES = 1_048_576 + + +class PolicyCreateRequest(PolicyCreationInput): + """Native body containing immutable policy content only.""" diff --git a/policyengine_api/fastapi_routes/v2/policies/response_models.py b/policyengine_api/fastapi_routes/v2/policies/response_models.py new file mode 100644 index 000000000..94784c831 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/policies/response_models.py @@ -0,0 +1,113 @@ +"""Strict HTTP response models for the native v2 policy API.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Generic, Literal, TypeVar +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, JsonValue + +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse +from policyengine_api.services.v2.policies.types import PolicyPage, PolicyRead + + +class StrictPolicyAPIModel(BaseModel): + """Strict request/response base with dataclass conversion support.""" + + model_config = ConfigDict( + extra="forbid", + allow_inf_nan=False, + from_attributes=True, + ) + + +class PolicyParameterValueItem(StrictPolicyAPIModel): + id: UUID + parameter_id: UUID + parameter_name: str + value: JsonValue + start_date: datetime + end_date: datetime | None + + +class PolicyItem(StrictPolicyAPIModel): + id: UUID + country_id: str + tax_benefit_model_id: UUID + created_at: datetime + updated_at: datetime + parameter_values: list[PolicyParameterValueItem] + + @classmethod + def from_read(cls, item: PolicyRead) -> "PolicyItem": + return cls.model_validate(item) + + +class PolicyDetailResult(StrictPolicyAPIModel): + item: PolicyItem + + +class PolicyPageResult(StrictPolicyAPIModel): + items: list[PolicyItem] + offset: int + limit: int + has_more: bool + + @classmethod + def from_page(cls, page: PolicyPage) -> "PolicyPageResult": + return cls( + items=[PolicyItem.from_read(item) for item in page.items], + offset=page.offset, + limit=page.limit, + has_more=page.has_more, + ) + + +ResultT = TypeVar("ResultT") + + +class PolicySuccessResponse(StrictPolicyAPIModel, Generic[ResultT]): + status: Literal["ok"] = "ok" + message: None = None + result: ResultT + + +class PolicyDetailResponse(PolicySuccessResponse[PolicyDetailResult]): + pass + + +class PolicyPageResponse(PolicySuccessResponse[PolicyPageResult]): + pass + + +POLICY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 400: { + "model": V2ErrorResponse, + "description": "The policy content or country selection is invalid.", + }, + 404: { + "model": V2ErrorResponse, + "description": "The selected policy or catalog does not exist.", + }, + 409: { + "model": V2ErrorResponse, + "description": "Immutable policy identity conflicts with stored state.", + }, + 413: { + "model": V2ErrorResponse, + "description": "The policy request body exceeds 1 MiB.", + }, + 422: { + "model": V2ErrorResponse, + "description": "The request does not match the policy schema.", + }, + 500: { + "model": V2ErrorResponse, + "description": "Stored policy integrity validation failed.", + }, + 503: { + "model": V2ErrorResponse, + "description": "Supabase policy persistence is unavailable.", + }, +} diff --git a/policyengine_api/fastapi_routes/v2/policies/routes.py b/policyengine_api/fastapi_routes/v2/policies/routes.py new file mode 100644 index 000000000..8e2a5d754 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/policies/routes.py @@ -0,0 +1,196 @@ +"""Native FastAPI routes for immutable v2 policies.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TypeVar +from uuid import UUID + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.exc import SQLAlchemyError +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.fastapi_routes.v2.policies.request_models import ( + MAXIMUM_POLICY_REQUEST_BYTES, + PolicyCreateRequest, +) +from policyengine_api.fastapi_routes.v2.policies.response_models import ( + POLICY_ERROR_RESPONSES, + PolicyDetailResponse, + PolicyDetailResult, + PolicyItem, + PolicyPageResponse, + PolicyPageResult, +) +from policyengine_api.fastapi_routes.v2.errors import ( + V2RequestTooLargeError, + v2_error_response, +) +from policyengine_api.services.v2.policies.types import NativePolicyCreationInput +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, + PolicyContentHashCollisionError, + PolicyCreationIntegrityError, + PolicyNotFoundError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2PolicyResourceService, +) +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + PolicyCollectionQuery, + PolicyCreateQuery, + PolicyDetailQuery, +) + + +async def enforce_policy_request_size(request: Request) -> None: + """Bound both declared and actual request bytes before service creation.""" + + content_length = request.headers.get("content-length") + if content_length is not None: + try: + declared_length = int(content_length) + except ValueError as error: + raise V2RequestTooLargeError( + "Policy request Content-Length is invalid" + ) from error + if declared_length > MAXIMUM_POLICY_REQUEST_BYTES: + raise V2RequestTooLargeError("Policy request body exceeds 1 MiB") + if len(await request.body()) > MAXIMUM_POLICY_REQUEST_BYTES: + raise V2RequestTooLargeError("Policy request body exceeds 1 MiB") + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2PolicyResourceService]: + if dependencies.v2_policy_service_factory is not None: + return dependencies.v2_policy_service_factory + from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_policy_service_factory, + ) + + return _default_v2_policy_service_factory + + +OperationT = TypeVar("OperationT") + + +def _policy_operation( + operation: Callable[[], OperationT], +) -> OperationT | JSONResponse: + try: + return operation() + except PolicyCatalogValidationError as error: + return v2_error_response(400, str(error)) + except (MetadataCatalogVersionNotFoundError, PolicyNotFoundError) as error: + return v2_error_response(404, str(error)) + except PolicyContentHashCollisionError: + return v2_error_response(409, "Policy content hash conflicts with storage") + except PolicyCreationIntegrityError: + return v2_error_response(500, "Stored policy integrity failed") + except (V2ConfigurationError, MetadataCatalogUnavailableError, SQLAlchemyError): + return v2_error_response(503, "V2 policy persistence is unavailable") + except Exception: # noqa: BLE001 - route must return a secret-safe typed error + return v2_error_response(500, "V2 policy operation failed") + + +def build_v2_policy_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build native policy routes without opening a database connection.""" + + router = APIRouter(prefix="/v2", responses=POLICY_ERROR_RESPONSES) + create_query = query_dependency(PolicyCreateQuery) + detail_query = query_dependency(PolicyDetailQuery) + collection_query = query_dependency(PolicyCollectionQuery) + service_factory = _service_factory(dependencies) + + @router.post( + "/policies", + response_model=PolicyDetailResponse, + status_code=201, + responses={ + 200: { + "model": PolicyDetailResponse, + "description": "Equivalent immutable content already exists.", + } + }, + summary="Create or find an immutable policy", + ) + def create_policy( + body: PolicyCreateRequest, + query: PolicyCreateQuery = Depends(create_query), + _size: None = Depends(enforce_policy_request_size), + ) -> PolicyDetailResponse | JSONResponse: + if body.country_id != query.country_id: + return v2_error_response( + 400, + "Body country_id must match query country_id", + ) + + def create() -> PolicyDetailResponse | JSONResponse: + result = service_factory().create_policy( + NativePolicyCreationInput( + **body.model_dump(), + policyengine_version=query.policyengine_version, + ) + ) + response = PolicyDetailResponse( + result=PolicyDetailResult(item=PolicyItem.from_read(result.item)) + ) + if result.created: + return response + return JSONResponse( + status_code=200, + content=response.model_dump(mode="json"), + ) + + return _policy_operation(create) + + @router.get( + "/policies/{policy_id}", + response_model=PolicyDetailResponse, + summary="Read one immutable policy", + ) + def get_policy( + policy_id: UUID, + query: PolicyDetailQuery = Depends(detail_query), + ) -> PolicyDetailResponse | JSONResponse: + def read() -> PolicyDetailResponse: + item = service_factory().get_policy( + country_id=query.country_id, + policy_id=policy_id, + ) + return PolicyDetailResponse( + result=PolicyDetailResult(item=PolicyItem.from_read(item)) + ) + + return _policy_operation(read) + + @router.get( + "/policies", + response_model=PolicyPageResponse, + summary="List immutable policies", + ) + def get_policies( + query: PolicyCollectionQuery = Depends(collection_query), + ) -> PolicyPageResponse | JSONResponse: + def read() -> PolicyPageResponse: + page = service_factory().list_policies( + country_id=query.country_id, + tax_benefit_model_id=query.tax_benefit_model_id, + offset=query.offset, + limit=query.limit, + ) + return PolicyPageResponse(result=PolicyPageResult.from_page(page)) + + return _policy_operation(read) + + return router diff --git a/policyengine_api/fastapi_routes/v2_metadata.py b/policyengine_api/fastapi_routes/v2/routes.py similarity index 55% rename from policyengine_api/fastapi_routes/v2_metadata.py rename to policyengine_api/fastapi_routes/v2/routes.py index 000eddcdd..68bc30fe0 100644 --- a/policyengine_api/fastapi_routes/v2_metadata.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -1,29 +1,35 @@ -"""Dormant, read-only API v2 metadata resource routes.""" +"""Compose the native API v2 policy and metadata routes.""" from __future__ import annotations from fastapi import APIRouter, Request from starlette.responses import JSONResponse -from policyengine_api.data.v2.catalog.schemas import MetadataErrorResponse from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -from policyengine_api.fastapi_routes.v2_metadata_geography import ( +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.metadata.geography_routes import ( build_v2_metadata_geography_router, ) -from policyengine_api.fastapi_routes.v2_metadata_models import ( +from policyengine_api.fastapi_routes.v2.metadata.model_routes import ( build_v2_metadata_model_router, ) -from policyengine_api.fastapi_routes.v2_metadata_parameters import ( +from policyengine_api.fastapi_routes.v2.metadata.parameter_routes import ( build_v2_metadata_parameter_router, ) +from policyengine_api.fastapi_routes.v2.policies.routes import build_v2_policy_router +from policyengine_api.fastapi_routes.v2.user_policies.routes import ( + build_v2_user_policy_router, +) -def build_v2_metadata_router( +def build_v2_router( dependencies: NativeRouteDependencies, ) -> APIRouter: """Build isolated resource routes without loading v2 configuration.""" router = APIRouter() + router.include_router(build_v2_user_policy_router(dependencies)) + router.include_router(build_v2_policy_router(dependencies)) router.include_router(build_v2_metadata_model_router(dependencies)) router.include_router(build_v2_metadata_parameter_router(dependencies)) router.include_router(build_v2_metadata_geography_router(dependencies)) @@ -47,44 +53,44 @@ def v2_preview_openapi(request: Request) -> JSONResponse: @router.get( "/v2", - response_model=MetadataErrorResponse, + response_model=V2ErrorResponse, status_code=404, include_in_schema=False, ) - def unsupported_v2_root() -> MetadataErrorResponse: - return MetadataErrorResponse(message="V2 metadata resource was not found") + def unsupported_v2_root() -> V2ErrorResponse: + return V2ErrorResponse(message="API v2 resource was not found") @router.api_route( "/v2", methods=["POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"], - response_model=MetadataErrorResponse, + response_model=V2ErrorResponse, status_code=405, include_in_schema=False, ) - def unsupported_v2_root_method() -> MetadataErrorResponse: - return MetadataErrorResponse(message="V2 metadata resources support GET only") + def unsupported_v2_root_method() -> V2ErrorResponse: + return V2ErrorResponse(message="API v2 resource does not support this method") @router.get( "/v2/{resource_path:path}", - response_model=MetadataErrorResponse, + response_model=V2ErrorResponse, status_code=404, include_in_schema=False, ) - def unsupported_resource(resource_path: str) -> MetadataErrorResponse: - return MetadataErrorResponse( - message=f"V2 metadata resource {resource_path!r} was not found" + def unsupported_resource(resource_path: str) -> V2ErrorResponse: + return V2ErrorResponse( + message=f"API v2 resource {resource_path!r} was not found" ) @router.api_route( "/v2/{resource_path:path}", methods=["POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"], - response_model=MetadataErrorResponse, + response_model=V2ErrorResponse, status_code=405, include_in_schema=False, ) - def unsupported_method(resource_path: str) -> MetadataErrorResponse: - return MetadataErrorResponse( - message=f"V2 metadata resource {resource_path!r} supports GET only" + def unsupported_method(resource_path: str) -> V2ErrorResponse: + return V2ErrorResponse( + message=f"API v2 resource {resource_path!r} does not support this method" ) return router diff --git a/policyengine_api/fastapi_routes/v2/user_policies/__init__.py b/policyengine_api/fastapi_routes/v2/user_policies/__init__.py new file mode 100644 index 000000000..180dc6780 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/user_policies/__init__.py @@ -0,0 +1 @@ +"""FastAPI adapters for mutable v2 user-policy associations.""" diff --git a/policyengine_api/fastapi_routes/v2/user_policies/request_models.py b/policyengine_api/fastapi_routes/v2/user_policies/request_models.py new file mode 100644 index 000000000..9292bc440 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/user_policies/request_models.py @@ -0,0 +1,14 @@ +"""Strict HTTP request models for native v2 user-policy associations.""" + +from policyengine_api.services.v2.user_policies.types import ( + UserPolicyCreationInput, + UserPolicyUpdateInput, +) + + +class UserPolicyCreateRequest(UserPolicyCreationInput): + """Association identity, immutable link fields, and presentation fields.""" + + +class UserPolicyPatchRequest(UserPolicyUpdateInput): + """Explicitly supplied mutable presentation fields.""" diff --git a/policyengine_api/fastapi_routes/v2/user_policies/response_models.py b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py new file mode 100644 index 000000000..caef44df3 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/user_policies/response_models.py @@ -0,0 +1,102 @@ +"""Strict HTTP response models for native v2 user-policy associations.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Generic, Literal, TypeVar +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse +from policyengine_api.services.v2.user_policies.types import ( + UserPolicyPage, + UserPolicyRead, +) +from policyengine_api.query_parameters import CountryId, ResourceId, UserId + + +class StrictUserPolicyAPIModel(BaseModel): + """Strict association contract with dataclass conversion support.""" + + model_config = ConfigDict(extra="forbid", from_attributes=True) + + +class UserPolicyItem(StrictUserPolicyAPIModel): + id: UUID + country_id: CountryId + user_id: UserId + policy_id: ResourceId + name: str | None + description: str | None + created_at: datetime + updated_at: datetime + + @classmethod + def from_read(cls, item: UserPolicyRead) -> "UserPolicyItem": + return cls.model_validate(item) + + +class UserPolicyDetailResult(StrictUserPolicyAPIModel): + item: UserPolicyItem + + +class UserPolicyPageResult(StrictUserPolicyAPIModel): + items: list[UserPolicyItem] + offset: int + limit: int + has_more: bool + + @classmethod + def from_page(cls, page: UserPolicyPage) -> "UserPolicyPageResult": + return cls( + items=[UserPolicyItem.from_read(item) for item in page.items], + offset=page.offset, + limit=page.limit, + has_more=page.has_more, + ) + + +ResultT = TypeVar("ResultT") + + +class UserPolicySuccessResponse(StrictUserPolicyAPIModel, Generic[ResultT]): + status: Literal["ok"] = "ok" + message: None = None + result: ResultT + + +class UserPolicyDetailResponse(UserPolicySuccessResponse[UserPolicyDetailResult]): + pass + + +class UserPolicyPageResponse(UserPolicySuccessResponse[UserPolicyPageResult]): + pass + + +USER_POLICY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 400: { + "model": V2ErrorResponse, + "description": "Association content or country selection is invalid.", + }, + 404: { + "model": V2ErrorResponse, + "description": "The selected user, policy, or association does not exist.", + }, + 409: { + "model": V2ErrorResponse, + "description": "Association state conflicts with stored state.", + }, + 422: { + "model": V2ErrorResponse, + "description": "The request does not match the association schema.", + }, + 500: { + "model": V2ErrorResponse, + "description": "The association operation could not be completed.", + }, + 503: { + "model": V2ErrorResponse, + "description": "Supabase association persistence is unavailable.", + }, +} diff --git a/policyengine_api/fastapi_routes/v2/user_policies/routes.py b/policyengine_api/fastapi_routes/v2/user_policies/routes.py new file mode 100644 index 000000000..7b4f78c77 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/user_policies/routes.py @@ -0,0 +1,207 @@ +"""Native FastAPI routes for mutable v2 user-policy associations.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TypeVar +from uuid import UUID + +from fastapi import APIRouter, Depends +from sqlalchemy.exc import SQLAlchemyError +from starlette.responses import JSONResponse, Response + +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.v2.user_policies.request_models import ( + UserPolicyCreateRequest, + UserPolicyPatchRequest, +) +from policyengine_api.fastapi_routes.v2.user_policies.response_models import ( + USER_POLICY_ERROR_RESPONSES, + UserPolicyDetailResponse, + UserPolicyDetailResult, + UserPolicyItem, + UserPolicyPageResponse, + UserPolicyPageResult, +) +from policyengine_api.fastapi_routes.v2.errors import v2_error_response +from policyengine_api.services.v2.user_policies.validators import ( + UserPolicyNotFoundError, + AssociationCountryConflictError, + AssociationPolicyNotFoundError, + AssociationUserNotFoundError, +) +from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2UserPolicyResourceService, +) +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + CountryQuery, + UserPolicyCollectionQuery, +) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2UserPolicyResourceService]: + if dependencies.v2_user_policy_service_factory is not None: + return dependencies.v2_user_policy_service_factory + from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_user_policy_service_factory, + ) + + return _default_v2_user_policy_service_factory + + +OperationT = TypeVar("OperationT") + + +def _association_operation( + operation: Callable[[], OperationT], +) -> OperationT | JSONResponse: + try: + return operation() + except AssociationCountryConflictError as error: + return v2_error_response(400, str(error)) + except ( + AssociationPolicyNotFoundError, + AssociationUserNotFoundError, + UserPolicyNotFoundError, + ) as error: + return v2_error_response(404, str(error)) + except (V2ConfigurationError, SQLAlchemyError): + return v2_error_response( + 503, + "V2 association persistence is unavailable", + ) + except Exception: # noqa: BLE001 - return a secret-safe typed error + return v2_error_response(500, "V2 association operation failed") + + +def build_v2_user_policy_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build native association routes without opening a database connection.""" + + router = APIRouter(prefix="/v2", responses=USER_POLICY_ERROR_RESPONSES) + country_query = query_dependency(CountryQuery) + collection_query = query_dependency(UserPolicyCollectionQuery) + service_factory = _service_factory(dependencies) + + @router.post( + "/user-policies", + response_model=UserPolicyDetailResponse, + status_code=201, + summary="Create a user-policy association", + description=( + "Creates a saved association for an existing v2 user UUID. The " + "UUID identifies a database row; this operation does not prove " + "that the caller controls that user and performs no authentication " + "or authorization check." + ), + ) + def create_user_policy( + body: UserPolicyCreateRequest, + query: CountryQuery = Depends(country_query), + ) -> UserPolicyDetailResponse | JSONResponse: + if body.country_id != query.country_id: + return v2_error_response( + 400, + "Body country_id must match query country_id", + ) + + def create() -> UserPolicyDetailResponse: + item = service_factory().create_user_policy(body) + return UserPolicyDetailResponse( + result=UserPolicyDetailResult(item=UserPolicyItem.from_read(item)) + ) + + return _association_operation(create) + + @router.get( + "/user-policies/{association_id}", + response_model=UserPolicyDetailResponse, + summary="Read one user-policy association", + ) + def get_user_policy( + association_id: UUID, + query: CountryQuery = Depends(country_query), + ) -> UserPolicyDetailResponse | JSONResponse: + def read() -> UserPolicyDetailResponse: + item = service_factory().get_user_policy( + country_id=query.country_id, + association_id=association_id, + ) + return UserPolicyDetailResponse( + result=UserPolicyDetailResult(item=UserPolicyItem.from_read(item)) + ) + + return _association_operation(read) + + @router.get( + "/user-policies", + response_model=UserPolicyPageResponse, + summary="List user-policy associations", + description=( + "Filters by a v2 user UUID. A match is not proof of caller control " + "and is not an authentication or authorization decision." + ), + ) + def get_user_policies( + query: UserPolicyCollectionQuery = Depends(collection_query), + ) -> UserPolicyPageResponse | JSONResponse: + def read() -> UserPolicyPageResponse: + page = service_factory().list_user_policies( + country_id=query.country_id, + user_id=query.user_id, + policy_id=query.policy_id, + offset=query.offset, + limit=query.limit, + ) + return UserPolicyPageResponse(result=UserPolicyPageResult.from_page(page)) + + return _association_operation(read) + + @router.patch( + "/user-policies/{association_id}", + response_model=UserPolicyDetailResponse, + summary="Update association presentation fields", + ) + def patch_user_policy_route( + association_id: UUID, + body: UserPolicyPatchRequest, + query: CountryQuery = Depends(country_query), + ) -> UserPolicyDetailResponse | JSONResponse: + def patch() -> UserPolicyDetailResponse: + item = service_factory().patch_user_policy( + country_id=query.country_id, + association_id=association_id, + association_input=body, + ) + return UserPolicyDetailResponse( + result=UserPolicyDetailResult(item=UserPolicyItem.from_read(item)) + ) + + return _association_operation(patch) + + @router.delete( + "/user-policies/{association_id}", + response_model=None, + status_code=204, + response_class=Response, + summary="Delete one user-policy association", + ) + def delete_user_policy_route( + association_id: UUID, + query: CountryQuery = Depends(country_query), + ) -> Response | JSONResponse: + def delete() -> Response: + service_factory().delete_user_policy( + country_id=query.country_id, + association_id=association_id, + ) + return Response(status_code=204) + + return _association_operation(delete) + + return router diff --git a/policyengine_api/migration_flags.py b/policyengine_api/migration_flags.py index 5cd95f20a..0d4cb44c2 100644 --- a/policyengine_api/migration_flags.py +++ b/policyengine_api/migration_flags.py @@ -30,6 +30,8 @@ class RouteImplementation(StrEnum): ) DB_WRITE_SOURCES = frozenset({"cloud_sql", "dual_write", "supabase"}) DB_READ_SOURCES = frozenset({"cloud_sql", "read_compare", "supabase"}) +V1_POLICY_WRITE_SOURCES = frozenset({"cloud_sql", "dual_write"}) +V1_POLICY_READ_SOURCES = frozenset({"cloud_sql"}) SIM_ENTRYPOINTS = frozenset({"old_gateway_direct", "cloud_run_simulation_entrypoint"}) SIM_COMPUTE_BACKENDS = frozenset( {"old_gateway", "v2_shadow", "v2_percent", "v2_primary"} @@ -143,6 +145,26 @@ def get_db_read(entity: str) -> str: return _read_choice(env_name, DEFAULT_DB_SOURCE, DB_READ_SOURCES) +def get_v1_policy_write_source() -> str: + """Select Cloud SQL alone or immediate Cloud SQL-to-Supabase mirroring.""" + + return _read_choice( + "DB_WRITE_POLICY", + DEFAULT_DB_SOURCE, + V1_POLICY_WRITE_SOURCES, + ) + + +def get_v1_policy_read_source() -> str: + """Require every v1 policy read to remain on Cloud SQL in Phase 10.""" + + return _read_choice( + "DB_READ_POLICY", + DEFAULT_DB_SOURCE, + V1_POLICY_READ_SOURCES, + ) + + def get_sim_compute(flow: str) -> str: env_name = f"SIM_COMPUTE_{flow.upper()}" return _read_choice( diff --git a/policyengine_api/migration_logging.py b/policyengine_api/migration_logging.py index 3de5ddfcb..701ea0886 100644 --- a/policyengine_api/migration_logging.py +++ b/policyengine_api/migration_logging.py @@ -29,6 +29,7 @@ "variables", } ) +V2_POLICY_RESOURCE_SEGMENTS = frozenset({"policies", "user-policies"}) def _is_v2_metadata_resource_read(method: str, path: str) -> bool: @@ -42,6 +43,17 @@ def _is_v2_metadata_resource_read(method: str, path: str) -> bool: ) +def _is_v2_policy_resource(method: str, path: str) -> bool: + if method not in {"GET", "POST", "PATCH", "DELETE"}: + return False + segments = [segment for segment in path.strip("/").split("/") if segment] + return ( + len(segments) >= 2 + and segments[0] == "v2" + and segments[1] in V2_POLICY_RESOURCE_SEGMENTS + ) + + def register_migration_request_logging(app: flask.Flask) -> None: """Register request IDs and migration logging for Flask.""" @@ -95,11 +107,22 @@ def log_migration_request( route_group = infer_route_group(path) is_v2_metadata_read = _is_v2_metadata_resource_read(method, path) + is_v2_policy_resource = _is_v2_policy_resource(method, path) + uses_explicit_v2_source = is_v2_metadata_read or is_v2_policy_resource migration_context = get_migration_log_context( route_group, route_impl=route_impl, - use_configured_db_sources=not is_v2_metadata_read, - db_read_source="supabase" if is_v2_metadata_read else None, + use_configured_db_sources=not uses_explicit_v2_source, + db_write_source=( + "supabase" + if is_v2_policy_resource and method in {"POST", "PATCH", "DELETE"} + else None + ), + db_read_source=( + "supabase" + if is_v2_metadata_read or (is_v2_policy_resource and method == "GET") + else None + ), ) logger.log_struct( diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index a19376b8c..f7403f58b 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -44,7 +44,7 @@ class RouteGroupConfig: ), RouteGroupConfig( name="policy", - path_segments=("policy", "policies", "user-policy"), + path_segments=("policy", "policies", "user-policy", "user-policies"), db_entity="policy", ), RouteGroupConfig( diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py new file mode 100644 index 000000000..a1fd12abf --- /dev/null +++ b/policyengine_api/query_parameters.py @@ -0,0 +1,209 @@ +"""Canonical query-parameter definitions shared by HTTP frameworks.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from types import UnionType +from typing import Annotated, Any, Literal, TypeVar, Union, get_args, get_origin +from uuid import UUID + +from pydantic import ( + AfterValidator, + BaseModel, + BeforeValidator, + ConfigDict, + Field, +) + +from policyengine_api.data.v2.catalog.catalog_selection import ( + validate_policyengine_version, +) + + +DEFAULT_QUERY_LIMIT = 100 +MAXIMUM_QUERY_LIMIT = 500 +MAXIMUM_USER_ID_LENGTH = 255 +SUPPORTED_COUNTRY_IDS = frozenset({"us", "uk"}) + + +def normalize_country_id(value: Any) -> Any: + """Lowercase a textual country ID before its supported-value check.""" + + return value.lower() if isinstance(value, str) else value + + +def validate_legacy_user_id(value: str) -> str: + """Reject an empty or whitespace-only legacy user identifier.""" + + if not value.strip(): + raise ValueError( + "legacy user_id must contain at least one non-whitespace character" + ) + return value + + +CountryId = Annotated[ + Literal["us", "uk"], + BeforeValidator(normalize_country_id), + Field(description="Supported PolicyEngine country ID"), +] +PolicyEngineVersion = Annotated[ + str, + Field(max_length=128, description="Canonical non-placeholder PEP 440 version"), + AfterValidator(validate_policyengine_version), +] +Offset = Annotated[int, Field(ge=0, description="Zero-based result offset")] +Limit = Annotated[ + int, + Field( + ge=1, + le=MAXIMUM_QUERY_LIMIT, + description="Maximum number of returned resources", + ), +] +ResourceId = Annotated[UUID, Field(description="Exact resource UUID")] +UserId = Annotated[ + UUID, + Field(description="V2 user UUID; does not prove caller control"), +] +LegacyUserId = Annotated[ + str, + Field( + min_length=1, + max_length=MAXIMUM_USER_ID_LENGTH, + description="Exact opaque v1 user identifier", + ), + AfterValidator(validate_legacy_user_id), +] + + +class StrictQueryParameters(BaseModel): + """Base for query contracts that reject every undeclared field.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class CountryQuery(StrictQueryParameters): + """Required country selection shared by country-scoped routes.""" + + country_id: CountryId + + +class CatalogQuery(CountryQuery): + """Country plus an optional exact PolicyEngine.py catalog version.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class PaginationQuery(StrictQueryParameters): + """Canonical bounded offset/limit pagination.""" + + offset: Offset = 0 + limit: Limit = DEFAULT_QUERY_LIMIT + + +class PolicyCreateQuery(CatalogQuery): + """Query contract for native immutable policy creation.""" + + +class PolicyDetailQuery(CountryQuery): + """Query contract for country-scoped policy detail reads.""" + + +class PolicyCollectionQuery(CountryQuery, PaginationQuery): + """Query contract for an exact-filtered policy collection.""" + + tax_benefit_model_id: ResourceId | None = None + + +class UserPolicyCollectionQuery(CountryQuery, PaginationQuery): + """Query contract for one v2 user's policy associations.""" + + user_id: UserId + policy_id: ResourceId | None = None + + +QueryParametersT = TypeVar("QueryParametersT", bound=StrictQueryParameters) + + +class DuplicateScalarQueryParameterError(ValueError): + """Raised when a scalar query field is supplied more than once.""" + + def __init__(self, parameter_name: str) -> None: + self.parameter_name = parameter_name + super().__init__( + f"scalar query parameter {parameter_name!r} must not be repeated" + ) + + +def _annotation_is_list(annotation: object) -> bool: + origin = get_origin(annotation) + if origin is list: + return True + if origin in (Union, UnionType): + return any(_annotation_is_list(member) for member in get_args(annotation)) + return False + + +def query_field_multiplicity( + model_type: type[StrictQueryParameters], +) -> Mapping[str, bool]: + """Return public query names mapped to whether repeated values are valid.""" + + return { + (field.alias or field_name): _annotation_is_list(field.annotation) + for field_name, field in model_type.model_fields.items() + } + + +def validate_scalar_query_multiplicity( + model_type: type[StrictQueryParameters], + items: Iterable[tuple[str, str]], +) -> None: + """Reject a repeated declared scalar while allowing declared list fields.""" + + multiplicity = query_field_multiplicity(model_type) + seen: set[str] = set() + for name, _value in items: + if name not in multiplicity or multiplicity[name]: + continue + if name in seen: + raise DuplicateScalarQueryParameterError(name) + seen.add(name) + + +def parse_query_items( + model_type: type[QueryParametersT], + items: Iterable[tuple[str, str]], +) -> QueryParametersT: + """Validate ordered query pairs through one canonical Pydantic schema.""" + + pairs = list(items) + validate_scalar_query_multiplicity(model_type, pairs) + multiplicity = query_field_multiplicity(model_type) + values: dict[str, object] = {} + for name, value in pairs: + if multiplicity.get(name, False): + values.setdefault(name, []) + list_values = values[name] + if isinstance(list_values, list): + list_values.append(value) + continue + values[name] = value + return model_type.model_validate(values) + + +def parse_multidict_query( + model_type: type[QueryParametersT], + query_input: object, +) -> QueryParametersT: + """Adapt a Flask/Werkzeug-style MultiDict to the canonical parser.""" + + items_method = getattr(query_input, "items", None) + if not callable(items_method): + raise TypeError("query input must provide an items method") + try: + items = items_method(multi=True) + except TypeError as error: + raise TypeError("query input must preserve repeated keys") from error + return parse_query_items(model_type, items) diff --git a/policyengine_api/readiness.py b/policyengine_api/readiness.py index 2ad374729..353df28e3 100644 --- a/policyengine_api/readiness.py +++ b/policyengine_api/readiness.py @@ -13,6 +13,28 @@ _ready = True +def validate_policy_runtime_configuration() -> None: + """Validate Phase 10 policy sources and conditionally require Supabase.""" + + from policyengine_api.data.v2.settings import ( + load_v2_runtime_database_settings, + ) + from policyengine_api.migration_flags import ( + RouteImplementation, + get_route_impl, + get_v1_policy_read_source, + get_v1_policy_write_source, + ) + + write_source = get_v1_policy_write_source() + get_v1_policy_read_source() + native_policy_routes = ( + get_route_impl("policy") is RouteImplementation.FASTAPI_NATIVE + ) + if write_source == "dual_write" or native_policy_routes: + load_v2_runtime_database_settings() + + def mark_not_ready() -> None: """Report not-ready — call before running the startup warmup.""" global _ready @@ -30,4 +52,11 @@ def mark_ready() -> None: def is_ready() -> bool: """Whether the service is warmed up and can serve a real request quickly.""" with _lock: - return _ready + warmed_up = _ready + if not warmed_up: + return False + try: + validate_policy_runtime_configuration() + except (RuntimeError, ValueError): + return False + return True diff --git a/policyengine_api/routes/policy_routes.py b/policyengine_api/routes/policy_routes.py index f857954a4..3918703e7 100644 --- a/policyengine_api/routes/policy_routes.py +++ b/policyengine_api/routes/policy_routes.py @@ -1,12 +1,34 @@ import json +import time from flask import Blueprint, Response, request from werkzeug.exceptions import BadRequest, NotFound from policyengine_api.data.v1_models import Policy, UserPolicy +from policyengine_api.data.v2.catalog.catalog_selection import ( + SUPPORTED_V2_COUNTRY_IDS, +) +from policyengine_api.gcp_logging import logger +from policyengine_api.migration_flags import ( + get_v1_policy_read_source, + get_v1_policy_write_source, +) +from policyengine_api.request_context import current_request_id from policyengine_api.response_factory import _make_error_response +from policyengine_api.services.policy_mirroring import ( + PolicyMirrorUnavailableError, + mirror_policy_after_commit, +) from policyengine_api.services.policy_service import PolicyService -from policyengine_api.services.user_policy_service import UserPolicyService +from policyengine_api.services.user_policy_service import ( + USER_POLICY_MUTABLE_FIELDS, + UserPolicyPersistenceError, + UserPolicyService, +) +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorUnavailableError, + mirror_pending_user_policy_events_after_commit, +) from policyengine_api.utils.payload_validators import ( validate_country, validate_set_policy_payload, @@ -17,6 +39,97 @@ user_policy_service = UserPolicyService() +def _should_mirror_to_v2(country_id: str, write_source: str) -> bool: + """Select immediate mirroring only for countries supported by API v2.""" + + return write_source == "dual_write" and country_id in SUPPORTED_V2_COUNTRY_IDS + + +def _policy_configuration_unavailable() -> Response: + return _make_error_response( + "Policy persistence configuration is unavailable.", + 503, + ) + + +def _policy_mirror_unavailable() -> Response: + return _make_error_response( + "V2 policy mirroring is unavailable; retry the same request.", + 503, + ) + + +def _user_policy_mirror_unavailable() -> Response: + return _make_error_response( + "V2 saved-policy mirroring is unavailable; retry the same request.", + 503, + include_status=False, + ) + + +def _safe_legacy_user_policy_id(value: object) -> int | None: + if ( + isinstance(value, int) + and not isinstance(value, bool) + and 0 <= value <= 2_147_483_647 + ): + return value + return None + + +def _user_policy_persistence_failure( + error: UserPolicyPersistenceError, + *, + operation: str, + country_id: str, + configured_write_source: str, + started_at: float, + legacy_user_policy_id: object | None = None, +) -> Response: + """Return and record a failure without serializing exception details.""" + + status_code = 503 if error.retryable else 500 + try: + logger.log_struct( + { + "message": "V1 saved-policy persistence failed", + "metric_name": "v1_user_policy_persistence_failures", + "metric_value": 1, + "resource": "user_policy", + "operation": operation, + "database_source": "cloud_sql", + "configured_write_source": configured_write_source, + "country_id": country_id, + "legacy_user_policy_id": _safe_legacy_user_policy_id( + legacy_user_policy_id + ), + "request_id": current_request_id(), + "outcome": "error", + "failure_category": error.category.value, + "http_status": status_code, + "duration_ms": round( + (time.perf_counter() - started_at) * 1000, + 3, + ), + }, + severity="ERROR", + ) + except Exception: + # A logging failure must not replace the persistence response. + pass + + message = ( + "Policy database is temporarily unavailable; please try again later." + if status_code == 503 + else "Internal database error; please try again later." + ) + return _make_error_response( + message, + status_code, + include_status=False, + ) + + def _serialize_policy(policy: Policy) -> dict: return { "id": policy.id, @@ -43,6 +156,11 @@ def get_policy(country_id: str, policy_id: int | str) -> Response: policy data in JSON format """ + try: + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() + # Specifically cast policy_id to an integer policy_id = int(policy_id) @@ -75,14 +193,31 @@ def set_policy(country_id: str) -> Response: if not is_payload_valid: raise BadRequest(f"Invalid JSON data; details: {message}") + try: + write_source = get_v1_policy_write_source() + except ValueError: + return _policy_configuration_unavailable() + label = payload.pop("label", None) policy_json = payload.pop("data", None) + mirror_to_v2 = _should_mirror_to_v2(country_id, write_source) - policy_id, message, is_existing_policy = policy_service.set_policy( + creation = policy_service.set_policy( country_id, label, policy_json, + prepare_for_mirroring=mirror_to_v2, ) + policy_id, message, is_existing_policy = creation + + if mirror_to_v2: + snapshot = getattr(creation, "snapshot", None) + if snapshot is None: + return _policy_mirror_unavailable() + try: + mirror_policy_after_commit(snapshot) + except PolicyMirrorUnavailableError: + return _policy_mirror_unavailable() response_body = dict( status="ok", @@ -100,6 +235,7 @@ def _serialize_user_policy(user_policy: UserPolicy) -> dict: return { column.name: getattr(user_policy, column.name) for column in UserPolicy.__table__.columns + if column.name != "mirror_revision" } @@ -110,6 +246,11 @@ def get_policy_search(country_id: str) -> Response: query = request.args.get("query", "") unique_only = request.args.get("unique_only", default=False, type=json.loads) + try: + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() + try: results = policy_service.search_policies( country_id, @@ -134,9 +275,9 @@ def get_policy_search(country_id: str) -> Response: status=200, mimetype="application/json", ) - except Exception as error: + except Exception: return _make_error_response( - f"Internal server error: {error}", + "Internal server error; please try again later.", 500, ) @@ -180,28 +321,59 @@ def set_user_policy(country_id: str) -> Response: } try: - creation = user_policy_service.create_or_get_user_policy(values) + write_source = get_v1_policy_write_source() + mirror_to_v2 = _should_mirror_to_v2(country_id, write_source) + if mirror_to_v2: + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() + + persistence_started_at = time.perf_counter() + try: + creation = user_policy_service.create_or_get_user_policy( + values, + record_mirror_event=mirror_to_v2, + ) user_policy = creation.user_policy - if not creation.created: - return Response( - json.dumps( - dict( - status="ok", - message=( - f"The reform #{reform_id} / baseline #{baseline_id} pair " - f"already exists for user {user_id}" - ), - result=dict(id=user_policy.id), - ) - ), - status=200, - mimetype="application/json", + except UserPolicyPersistenceError as error: + return _user_policy_persistence_failure( + error, + operation="create", + country_id=country_id, + configured_write_source=write_source, + started_at=persistence_started_at, + ) + + if mirror_to_v2: + if creation.mirror_revision is None: + return _user_policy_mirror_unavailable() + try: + mirror_pending_user_policy_events_after_commit( + country_id, + creation.user_policy.id, + through_revision=creation.mirror_revision, + event_service=user_policy_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, ) - except Exception as error: - return _make_error_response( - f"Internal database error: {error}; please try again later.", - 500, - include_status=False, + except UserPolicyMirrorUnavailableError: + return _user_policy_mirror_unavailable() + except Exception: + return _user_policy_mirror_unavailable() + + if not creation.created: + return Response( + json.dumps( + dict( + status="ok", + message=( + f"The reform #{reform_id} / baseline #{baseline_id} pair " + f"already exists for user {user_id}" + ), + result=dict(id=user_policy.id), + ) + ), + status=200, + mimetype="application/json", ) return Response( @@ -219,8 +391,12 @@ def set_user_policy(country_id: str) -> Response: @policy_bp.route("//user-policy/", methods=["GET"]) @validate_country -def get_user_policy(country_id: str, user_id: str) -> dict: +def get_user_policy(country_id: str, user_id: str) -> dict | Response: """Fetch all saved policies for a user.""" + try: + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() user_policies = user_policy_service.list_user_policies(country_id, user_id) return dict( status="ok", @@ -229,21 +405,7 @@ def get_user_policy(country_id: str, user_id: str) -> dict: ) -UPDATE_USER_POLICY_ALLOWED_FIELDS = frozenset( - { - "reform_label", - "baseline_label", - "year", - "geography", - "dataset", - "number_of_provisions", - "api_version", - "added_date", - "updated_date", - "budgetary_impact", - "type", - } -) +UPDATE_USER_POLICY_ALLOWED_FIELDS = USER_POLICY_MUTABLE_FIELDS @policy_bp.route("//user-policy", methods=["PUT"]) @@ -277,25 +439,54 @@ def update_user_policy(country_id: str) -> Response: ) try: - user_policy = user_policy_service.update_user_policy( + write_source = get_v1_policy_write_source() + mirror_to_v2 = _should_mirror_to_v2(country_id, write_source) + if mirror_to_v2: + get_v1_policy_read_source() + except ValueError: + return _policy_configuration_unavailable() + + persistence_started_at = time.perf_counter() + try: + update = user_policy_service.update_user_policy( country_id, user_policy_id, payload, + record_mirror_event=mirror_to_v2, ) - except Exception as error: - return _make_error_response( - f"Internal database error: {error}; please try again later.", - 500, - include_status=False, + except UserPolicyPersistenceError as error: + return _user_policy_persistence_failure( + error, + operation="update", + country_id=country_id, + configured_write_source=write_source, + started_at=persistence_started_at, + legacy_user_policy_id=user_policy_id, ) - if user_policy is None: + if update is None: return _make_error_response( f"User policy #{user_policy_id} not found.", 404, include_status=False, ) + if mirror_to_v2: + if update.mirror_revision is None: + return _user_policy_mirror_unavailable() + try: + mirror_pending_user_policy_events_after_commit( + country_id, + update.user_policy.id, + through_revision=update.mirror_revision, + event_service=user_policy_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + ) + except UserPolicyMirrorUnavailableError: + return _user_policy_mirror_unavailable() + except Exception: + return _user_policy_mirror_unavailable() + return Response( json.dumps( dict( diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py new file mode 100644 index 000000000..84d055b95 --- /dev/null +++ b/policyengine_api/services/policy_mirroring.py @@ -0,0 +1,149 @@ +"""Immediate, observable mirroring of committed v1 policies into v2.""" + +from __future__ import annotations + +from collections.abc import Callable +import time +from typing import Protocol + +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.services.v2.policies.types import ( + LegacyPolicyPersistenceResult, + LegacyPolicySnapshot, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, + LegacyPolicyTranslationError, + PolicyCatalogValidationError, + PolicyContentHashCollisionError, + PolicyCreationIntegrityError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.gcp_logging import logger + + +class LegacyPolicyMirror(Protocol): + """Supabase transaction service used after a v1 commit.""" + + def mirror_legacy_policy( + self, + snapshot: LegacyPolicySnapshot, + ) -> LegacyPolicyPersistenceResult: ... + + +class PolicyMirrorUnavailableError(RuntimeError): + """Raised when a committed v1 policy could not be mirrored immediately.""" + + +def _default_mirror_factory() -> LegacyPolicyMirror: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, + ) + from policyengine_api.services.v2.policies.services import V2PolicyService + + return V2PolicyService(PolicyDatabaseSession(get_v2_session_factory())) + + +def _failure_category(error: Exception) -> str: + if isinstance(error, V2ConfigurationError): + return "configuration" + if isinstance( + error, + ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, + PolicyCatalogValidationError, + LegacyPolicyTranslationError, + ), + ): + return "catalog_or_translation" + if isinstance( + error, + ( + LegacyPolicyMappingIntegrityError, + PolicyContentHashCollisionError, + PolicyCreationIntegrityError, + ), + ): + return "integrity" + if isinstance(error, SQLAlchemyError): + return "database" + return "unexpected" + + +def _log_mirror_event( + *, + snapshot: LegacyPolicySnapshot, + started_at: float, + outcome: str, + actual_write_sources: list[str], + destination_policy_id: object | None = None, + failure_category: str | None = None, + policy_created: bool | None = None, + mapping_created: bool | None = None, +) -> None: + logger.log_struct( + { + "message": "V1 policy immediate mirror completed", + "metric_name": "v1_policy_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": actual_write_sources, + "country_id": snapshot.country_id, + "legacy_policy_id": snapshot.legacy_policy_id, + "destination_policy_id": ( + str(destination_policy_id) + if destination_policy_id is not None + else None + ), + "outcome": outcome, + "failure_category": failure_category, + "policy_created": policy_created, + "mapping_created": mapping_created, + "duration_ms": round((time.perf_counter() - started_at) * 1000, 3), + }, + severity="INFO" if outcome == "ok" else "ERROR", + ) + + +def mirror_policy_after_commit( + snapshot: LegacyPolicySnapshot, + *, + mirror_factory: Callable[[], LegacyPolicyMirror] | None = None, +) -> LegacyPolicyPersistenceResult: + """Require the Supabase policy and mapping transaction before v1 success.""" + + started_at = time.perf_counter() + try: + result = (mirror_factory or _default_mirror_factory)().mirror_legacy_policy( + snapshot + ) + except Exception as error: + _log_mirror_event( + snapshot=snapshot, + started_at=started_at, + outcome="error", + actual_write_sources=["cloud_sql"], + failure_category=_failure_category(error), + ) + raise PolicyMirrorUnavailableError( + "The committed v1 policy could not be mirrored to v2" + ) from error + + _log_mirror_event( + snapshot=snapshot, + started_at=started_at, + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + destination_policy_id=result.policy_id, + policy_created=result.policy_created, + mapping_created=result.mapping_created, + ) + return result diff --git a/policyengine_api/services/policy_service.py b/policyengine_api/services/policy_service.py index 728565882..b8b011f43 100644 --- a/policyengine_api/services/policy_service.py +++ b/policyengine_api/services/policy_service.py @@ -1,5 +1,8 @@ from __future__ import annotations +import copy +from dataclasses import dataclass +from collections.abc import Iterator from typing import Any from sqlalchemy import select @@ -8,9 +11,27 @@ from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import Policy +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot from policyengine_api.utils import hash_object +@dataclass(frozen=True) +class PolicySetResult: + """Existing v1 return values and an optional detached mirror snapshot.""" + + policy_id: int + message: str + is_existing_policy: bool + snapshot: LegacyPolicySnapshot | None + + def __iter__(self) -> Iterator[int | str | bool]: + """Preserve the established three-value internal unpacking interface.""" + + yield self.policy_id + yield self.message + yield self.is_existing_policy + + class PolicyService: """Policy operations with service-owned ORM transaction boundaries.""" @@ -63,6 +84,25 @@ def get_policy_json( policy = self.get_policy(country_id, policy_id) return None if policy is None else policy.policy_json + def get_policy_snapshot( + self, + country_id: str, + policy_id: int, + ) -> LegacyPolicySnapshot | None: + """Return detached fields required to mirror one existing v1 policy.""" + + policy = self.get_policy(country_id, policy_id) + if policy is None: + return None + return LegacyPolicySnapshot( + country_id=policy.country_id, + legacy_policy_id=policy.id, + label=policy.label, + api_version=policy.api_version, + policy_json=copy.deepcopy(policy.policy_json), + source_policy_hash=policy.policy_hash, + ) + def search_policies( self, country_id: str, @@ -96,20 +136,40 @@ def set_policy( country_id: str, label: str | None, policy_json: dict, - ) -> tuple[int, str, bool]: + *, + prepare_for_mirroring: bool = False, + ) -> PolicySetResult: country_id = country_id.lower() if country_id not in COUNTRY_PACKAGE_VERSIONS: raise ValueError(f"Invalid country_id: {country_id}") policy_hash = hash_object(policy_json) with self._sessions.begin() as session: - return self._set_policy( + policy, message, is_existing_policy = self._set_policy( session, country_id, label, policy_json, policy_hash, ) + snapshot = ( + LegacyPolicySnapshot( + country_id=policy.country_id, + legacy_policy_id=policy.id, + label=policy.label, + api_version=policy.api_version, + policy_json=copy.deepcopy(policy.policy_json), + source_policy_hash=policy.policy_hash, + ) + if prepare_for_mirroring + else None + ) + return PolicySetResult( + policy_id=policy.id, + message=message, + is_existing_policy=is_existing_policy, + snapshot=snapshot, + ) def _set_policy( self, @@ -118,7 +178,7 @@ def _set_policy( label: str | None, policy_json: dict, policy_hash: str, - ) -> tuple[int, str, bool]: + ) -> tuple[Policy, str, bool]: existing = self._get_unique_policy_with_label( session, country_id, @@ -126,7 +186,7 @@ def _set_policy( label or None, ) if existing is not None: - return existing.id, "Policy already exists", True + return existing, "Policy already exists", True policy = Policy( country_id=country_id, @@ -137,7 +197,7 @@ def _set_policy( ) session.add(policy) session.flush() - return policy.id, "Policy created", False + return policy, "Policy created", False def _create_new_policy( self, diff --git a/policyengine_api/services/user_policy_mirroring.py b/policyengine_api/services/user_policy_mirroring.py new file mode 100644 index 000000000..5cf316ace --- /dev/null +++ b/policyengine_api/services/user_policy_mirroring.py @@ -0,0 +1,306 @@ +"""Immediate mirroring of committed v1 saved policies into v2 associations.""" + +from __future__ import annotations + +from collections.abc import Callable +import time +from typing import Protocol + +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, + LegacyPolicyTranslationError, + PolicyCatalogValidationError, + PolicyContentHashCollisionError, + PolicyCreationIntegrityError, +) +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.validators import ( + LegacyUserPolicyIntegrityError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.gcp_logging import logger +from policyengine_api.services.policy_service import PolicyService +from policyengine_api.services.user_policy_service import ( + PendingUserPolicyMirrorEvent, + UserPolicyService, +) + + +class LegacyUserPolicyMirror(Protocol): + def mirror_legacy_user_policy( + self, + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str], + ) -> LegacyUserPolicyPersistenceResult: ... + + +class UserPolicyMirrorUnavailableError(RuntimeError): + """Raised when a committed saved policy could not be mirrored immediately.""" + + +def _default_mirror_factory() -> LegacyUserPolicyMirror: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, + ) + from policyengine_api.services.v2.user_policies.services import ( + V2UserPolicyService, + ) + + return V2UserPolicyService(UserPolicyDatabaseSession(get_v2_session_factory())) + + +def _failure_category(error: Exception) -> str: + if isinstance(error, V2ConfigurationError): + return "configuration" + if isinstance( + error, + ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, + PolicyCatalogValidationError, + LegacyPolicyTranslationError, + ), + ): + return "catalog_or_translation" + if isinstance( + error, + ( + LegacyPolicyMappingIntegrityError, + LegacyUserPolicyIntegrityError, + PolicyContentHashCollisionError, + PolicyCreationIntegrityError, + ), + ): + return "integrity" + if isinstance(error, SQLAlchemyError): + return "database" + return "unexpected" + + +def _log_mirror_operation( + *, + country_id: str, + legacy_user_policy_id: int, + source_revision: int | None, + requested_through_revision: int, + started_at: float, + outcome: str, + actual_write_sources: list[str], + result: LegacyUserPolicyPersistenceResult | None = None, + failure_category: str | None = None, +) -> None: + logger.log_struct( + { + "message": "V1 saved-policy immediate mirror completed", + "metric_name": "v1_user_policy_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": actual_write_sources, + "country_id": country_id, + "legacy_user_policy_id": legacy_user_policy_id, + "source_revision": source_revision, + "requested_through_revision": requested_through_revision, + "destination_association_id": ( + str(result.association_id) if result is not None else None + ), + "destination_policy_id": ( + str(result.policy_id) if result is not None else None + ), + "outcome": outcome, + "failure_category": failure_category, + "association_created": ( + result.association_created if result is not None else None + ), + "association_updated": ( + result.association_updated if result is not None else None + ), + "mapping_created": result.mapping_created if result is not None else None, + "duration_ms": round((time.perf_counter() - started_at) * 1000, 3), + }, + severity="INFO" if outcome == "ok" else "ERROR", + ) + + +def _run_user_policy_mirror( + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str], + mirror_factory: Callable[[], LegacyUserPolicyMirror] | None, +) -> LegacyUserPolicyPersistenceResult: + return (mirror_factory or _default_mirror_factory)().mirror_legacy_user_policy( + snapshot, + reform_snapshot, + source_revision=source_revision, + changed_fields=changed_fields, + ) + + +def mirror_user_policy_after_commit( + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str] = frozenset(), + mirror_factory: Callable[[], LegacyUserPolicyMirror] | None = None, +) -> LegacyUserPolicyPersistenceResult: + """Require one complete Supabase association transaction before v1 success.""" + + started_at = time.perf_counter() + try: + result = _run_user_policy_mirror( + snapshot, + reform_snapshot, + source_revision=source_revision, + changed_fields=changed_fields, + mirror_factory=mirror_factory, + ) + except Exception as error: + _log_mirror_operation( + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + source_revision=source_revision, + requested_through_revision=source_revision, + started_at=started_at, + outcome="error", + actual_write_sources=["cloud_sql"], + failure_category=_failure_category(error), + ) + raise UserPolicyMirrorUnavailableError( + "The committed v1 saved policy could not be mirrored to v2" + ) from error + + _log_mirror_operation( + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + source_revision=source_revision, + requested_through_revision=source_revision, + started_at=started_at, + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + result=result, + ) + return result + + +def mirror_pending_user_policy_events_after_commit( + country_id: str, + legacy_user_policy_id: int, + *, + through_revision: int, + event_service: UserPolicyService | None = None, + reform_snapshot_loader: Callable[[str, int], LegacyPolicySnapshot | None] + | None = None, + mirror_factory: Callable[[], LegacyUserPolicyMirror] | None = None, +) -> LegacyUserPolicyPersistenceResult: + """Synchronously apply retained source events through one request's revision.""" + + selected_event_service = event_service or UserPolicyService() + selected_reform_loader = ( + reform_snapshot_loader or PolicyService().get_policy_snapshot + ) + request_started_at = time.perf_counter() + active_event: PendingUserPolicyMirrorEvent | None = None + event_started_at: dict[int, float] = {} + destination_results: dict[int, LegacyUserPolicyPersistenceResult] = {} + + def process( + event: PendingUserPolicyMirrorEvent, + ) -> LegacyUserPolicyPersistenceResult: + nonlocal active_event + + active_event = event + event_started_at[event.event_id] = time.perf_counter() + reform_snapshot = selected_reform_loader( + event.snapshot.country_id, + event.snapshot.reform_id, + ) + if reform_snapshot is None: + raise UserPolicyMirrorUnavailableError( + "The saved policy references an unavailable reform policy" + ) + result = _run_user_policy_mirror( + event.snapshot, + reform_snapshot, + source_revision=event.source_revision, + changed_fields=event.changed_fields, + mirror_factory=mirror_factory, + ) + destination_results[event.event_id] = result + return result + + def after_processed_commit( + event: PendingUserPolicyMirrorEvent, + result: LegacyUserPolicyPersistenceResult, + ) -> None: + nonlocal active_event + + _log_mirror_operation( + country_id=event.snapshot.country_id, + legacy_user_policy_id=event.snapshot.legacy_user_policy_id, + source_revision=event.source_revision, + requested_through_revision=through_revision, + started_at=event_started_at.pop(event.event_id), + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + result=result, + ) + destination_results.pop(event.event_id, None) + active_event = None + + try: + return selected_event_service.process_pending_mirror_events( + country_id, + legacy_user_policy_id, + through_revision=through_revision, + processor=process, + after_processed_commit=after_processed_commit, + ) + except Exception as error: + event = active_event + result = destination_results.get(event.event_id) if event is not None else None + failure = error + if isinstance(error, UserPolicyMirrorUnavailableError) and isinstance( + error.__cause__, Exception + ): + failure = error.__cause__ + _log_mirror_operation( + country_id=(event.snapshot.country_id if event is not None else country_id), + legacy_user_policy_id=( + event.snapshot.legacy_user_policy_id + if event is not None + else legacy_user_policy_id + ), + source_revision=event.source_revision if event is not None else None, + requested_through_revision=through_revision, + started_at=( + event_started_at.get(event.event_id, request_started_at) + if event is not None + else request_started_at + ), + outcome="error", + actual_write_sources=( + ["cloud_sql", "supabase"] if result is not None else ["cloud_sql"] + ), + result=result, + failure_category=_failure_category(failure), + ) + raise UserPolicyMirrorUnavailableError( + "The committed v1 saved policy event could not be mirrored to v2" + ) from error diff --git a/policyengine_api/services/user_policy_service.py b/policyengine_api/services/user_policy_service.py index 07339310f..ab0c4e7d3 100644 --- a/policyengine_api/services/user_policy_service.py +++ b/policyengine_api/services/user_policy_service.py @@ -1,13 +1,29 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass +from datetime import datetime, timezone +from enum import StrEnum from typing import Any, Mapping from sqlalchemy import select +from sqlalchemy.exc import ( + IntegrityError, + OperationalError, + SQLAlchemyError, + TimeoutError as SQLAlchemyTimeoutError, +) from sqlalchemy.orm import Session, sessionmaker from policyengine_api.data.orm import get_v1_session_factory -from policyengine_api.data.v1_models import UserPolicy +from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.transformations import ( + fingerprint_legacy_user_policy, +) USER_POLICY_IDENTITY_FIELDS = ( @@ -21,12 +37,87 @@ "baseline_label", "dataset", ) +USER_POLICY_MIRROR_PAYLOAD_SCHEMA_VERSION = 1 +USER_POLICY_MUTABLE_FIELDS = frozenset( + { + "reform_label", + "baseline_label", + "year", + "geography", + "dataset", + "number_of_provisions", + "api_version", + "added_date", + "updated_date", + "budgetary_impact", + "type", + } +) + + +class UserPolicyMirrorEventIntegrityError(RuntimeError): + """Raised when durable saved-policy mirror input is invalid or inconsistent.""" + + +class UserPolicyPersistenceFailureCategory(StrEnum): + TIMEOUT = "timeout" + UNAVAILABLE = "unavailable" + INTEGRITY = "integrity" + DATABASE = "database" + UNEXPECTED = "unexpected" + + +class UserPolicyPersistenceError(RuntimeError): + """Domain error containing only safe persistence-failure attributes.""" + + def __init__(self, category: UserPolicyPersistenceFailureCategory) -> None: + super().__init__("Saved-policy persistence failed") + self.category = UserPolicyPersistenceFailureCategory(category) + + @property + def retryable(self) -> bool: + return self.category in { + UserPolicyPersistenceFailureCategory.TIMEOUT, + UserPolicyPersistenceFailureCategory.UNAVAILABLE, + } + + @classmethod + def from_exception(cls, error: Exception) -> UserPolicyPersistenceError: + if isinstance(error, SQLAlchemyTimeoutError): + return cls(UserPolicyPersistenceFailureCategory.TIMEOUT) + if isinstance(error, OperationalError): + return cls(UserPolicyPersistenceFailureCategory.UNAVAILABLE) + if isinstance(error, IntegrityError): + return cls(UserPolicyPersistenceFailureCategory.INTEGRITY) + if isinstance(error, SQLAlchemyError): + return cls(UserPolicyPersistenceFailureCategory.DATABASE) + return cls(UserPolicyPersistenceFailureCategory.UNEXPECTED) + + +@dataclass(frozen=True) +class PendingUserPolicyMirrorEvent: + event_id: int + source_revision: int + event_type: str + snapshot: LegacyUserPolicySnapshot + changed_fields: frozenset[str] + source_fingerprint_sha256: str @dataclass(frozen=True) class UserPolicyCreateResult: user_policy: UserPolicy created: bool + snapshot: LegacyUserPolicySnapshot | None + mirror_revision: int | None = None + + +@dataclass(frozen=True) +class UserPolicyUpdateResult: + user_policy: UserPolicy + snapshot: LegacyUserPolicySnapshot | None + changed_fields: frozenset[str] + mirror_revision: int | None = None class UserPolicyService: @@ -42,35 +133,178 @@ def __init__( def _sessions(self) -> sessionmaker[Session]: return self._injected_session_factory or get_v1_session_factory() + @staticmethod + def _snapshot(user_policy: UserPolicy) -> LegacyUserPolicySnapshot: + return LegacyUserPolicySnapshot( + country_id=user_policy.country_id, + legacy_user_policy_id=user_policy.id, + reform_id=user_policy.reform_id, + reform_label=user_policy.reform_label, + baseline_id=user_policy.baseline_id, + baseline_label=user_policy.baseline_label, + user_id=user_policy.user_id, + year=user_policy.year, + geography=user_policy.geography, + dataset=user_policy.dataset, + number_of_provisions=user_policy.number_of_provisions, + api_version=user_policy.api_version, + added_date=user_policy.added_date, + updated_date=user_policy.updated_date, + budgetary_impact=user_policy.budgetary_impact, + type=user_policy.type, + ) + @staticmethod def _find_matching_user_policy( session: Session, values: Mapping[str, Any], + *, + lock: bool, ) -> UserPolicy | None: - return session.scalar( - select(UserPolicy).where( - *( - getattr(UserPolicy, field) == values[field] - for field in USER_POLICY_IDENTITY_FIELDS - ) + statement = select(UserPolicy).where( + *( + getattr(UserPolicy, field) == values[field] + for field in USER_POLICY_IDENTITY_FIELDS + ) + ) + if lock: + statement = statement.with_for_update() + return session.scalar(statement) + + @classmethod + def _record_mirror_event( + cls, + session: Session, + user_policy: UserPolicy, + *, + event_type: str, + changed_fields: frozenset[str], + ) -> tuple[int, LegacyUserPolicySnapshot]: + user_policy.mirror_revision += 1 + snapshot = cls._snapshot(user_policy) + event = UserPolicyMirrorEvent( + country_id=user_policy.country_id, + legacy_user_policy_id=user_policy.id, + source_revision=user_policy.mirror_revision, + event_type=event_type, + payload_schema_version=USER_POLICY_MIRROR_PAYLOAD_SCHEMA_VERSION, + payload_json={ + "snapshot": snapshot.model_dump(mode="json"), + "changed_fields": sorted(changed_fields), + }, + source_fingerprint_sha256=fingerprint_legacy_user_policy(snapshot), + ) + session.add_all((user_policy, event)) + session.flush() + return user_policy.mirror_revision, snapshot + + @staticmethod + def _decode_mirror_event( + event: UserPolicyMirrorEvent, + ) -> PendingUserPolicyMirrorEvent: + if event.payload_schema_version != USER_POLICY_MIRROR_PAYLOAD_SCHEMA_VERSION: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event payload version is unsupported" + ) + payload = event.payload_json + if not isinstance(payload, dict) or set(payload) != { + "snapshot", + "changed_fields", + }: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event payload has an invalid shape" + ) + changed_fields = payload["changed_fields"] + if not isinstance(changed_fields, list) or not all( + isinstance(field, str) for field in changed_fields + ): + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event changed fields are invalid" + ) + if ( + changed_fields != sorted(set(changed_fields)) + or not set(changed_fields) <= USER_POLICY_MUTABLE_FIELDS + ): + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event changed fields are unsupported" + ) + try: + snapshot = LegacyUserPolicySnapshot.model_validate(payload["snapshot"]) + except (TypeError, ValueError) as error: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event snapshot is invalid" + ) from error + if ( + snapshot.country_id != event.country_id + or snapshot.legacy_user_policy_id != event.legacy_user_policy_id + ): + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event source identity conflicts with its payload" + ) + fingerprint = fingerprint_legacy_user_policy(snapshot) + if fingerprint != event.source_fingerprint_sha256: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event fingerprint conflicts with its payload" + ) + if event.event_type not in {"create", "update"}: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror event type is unsupported" ) + if event.event_type == "create" and changed_fields: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy create mirror event contains changed fields" + ) + if event.event_type == "update" and not changed_fields: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy update mirror event has no changed fields" + ) + return PendingUserPolicyMirrorEvent( + event_id=event.id, + source_revision=event.source_revision, + event_type=event.event_type, + snapshot=snapshot, + changed_fields=frozenset(changed_fields), + source_fingerprint_sha256=fingerprint, ) def create_or_get_user_policy( self, values: Mapping[str, Any], + *, + record_mirror_event: bool = False, ) -> UserPolicyCreateResult: - with self._sessions.begin() as session: - user_policy = self._find_matching_user_policy(session, values) - created = user_policy is None - if user_policy is None: - user_policy = UserPolicy(**values) - session.add(user_policy) - session.flush() - return UserPolicyCreateResult( - user_policy=user_policy, - created=created, - ) + try: + with self._sessions.begin() as session: + user_policy = self._find_matching_user_policy( + session, + values, + lock=record_mirror_event, + ) + created = user_policy is None + if user_policy is None: + user_policy = UserPolicy(**values) + session.add(user_policy) + session.flush() + mirror_revision = None + snapshot = None + if record_mirror_event: + mirror_revision, snapshot = self._record_mirror_event( + session, + user_policy, + event_type="create", + changed_fields=frozenset(), + ) + result = UserPolicyCreateResult( + user_policy=user_policy, + created=created, + snapshot=snapshot, + mirror_revision=mirror_revision, + ) + except UserPolicyPersistenceError: + raise + except Exception as error: + raise UserPolicyPersistenceError.from_exception(error) from error + return result def list_user_policies( self, @@ -92,16 +326,114 @@ def update_user_policy( country_id: str, user_policy_id: int, values: Mapping[str, Any], - ) -> UserPolicy | None: - with self._sessions.begin() as session: - user_policy = session.scalar( - select(UserPolicy).where( + *, + record_mirror_event: bool = False, + ) -> UserPolicyUpdateResult | None: + try: + with self._sessions.begin() as session: + statement = select(UserPolicy).where( UserPolicy.id == user_policy_id, UserPolicy.country_id == country_id, ) - ) - if user_policy is None: - return None - for field, value in values.items(): - setattr(user_policy, field, value) - return user_policy + if record_mirror_event: + statement = statement.with_for_update() + user_policy = session.scalar(statement) + if user_policy is None: + return None + for field, value in values.items(): + setattr(user_policy, field, value) + session.flush() + changed_fields = frozenset(values) + mirror_revision = None + snapshot = None + if record_mirror_event: + mirror_revision, snapshot = self._record_mirror_event( + session, + user_policy, + event_type="update", + changed_fields=changed_fields, + ) + result = UserPolicyUpdateResult( + user_policy=user_policy, + snapshot=snapshot, + changed_fields=changed_fields, + mirror_revision=mirror_revision, + ) + except UserPolicyPersistenceError: + raise + except Exception as error: + raise UserPolicyPersistenceError.from_exception(error) from error + return result + + def process_pending_mirror_events( + self, + country_id: str, + legacy_user_policy_id: int, + *, + through_revision: int, + processor: Callable[ + [PendingUserPolicyMirrorEvent], + LegacyUserPolicyPersistenceResult, + ], + after_processed_commit: Callable[ + [ + PendingUserPolicyMirrorEvent, + LegacyUserPolicyPersistenceResult, + ], + None, + ] + | None = None, + ) -> LegacyUserPolicyPersistenceResult: + """Process retained source events in order through one request's revision.""" + + latest_result: LegacyUserPolicyPersistenceResult | None = None + while True: + with self._sessions.begin() as session: + event = session.scalar( + select(UserPolicyMirrorEvent) + .where( + UserPolicyMirrorEvent.country_id == country_id, + UserPolicyMirrorEvent.legacy_user_policy_id + == legacy_user_policy_id, + UserPolicyMirrorEvent.source_revision <= through_revision, + UserPolicyMirrorEvent.processed_at.is_(None), + ) + .order_by(UserPolicyMirrorEvent.source_revision) + .limit(1) + .with_for_update() + ) + if event is None: + break + pending = self._decode_mirror_event(event) + processed_result = processor(pending) + event.processed_at = datetime.now(timezone.utc).replace(tzinfo=None) + session.add(event) + session.flush() + latest_result = processed_result + if after_processed_commit is not None: + after_processed_commit(pending, processed_result) + if pending.source_revision == through_revision: + break + if latest_result is None: + with self._sessions.begin() as session: + event = session.scalar( + select(UserPolicyMirrorEvent) + .where( + UserPolicyMirrorEvent.country_id == country_id, + UserPolicyMirrorEvent.legacy_user_policy_id + == legacy_user_policy_id, + UserPolicyMirrorEvent.source_revision == through_revision, + UserPolicyMirrorEvent.processed_at.is_not(None), + ) + .with_for_update() + ) + if event is None: + raise UserPolicyMirrorEventIntegrityError( + "saved-policy mirror request has no retained event" + ) + pending = self._decode_mirror_event(event) + processed_result = processor(pending) + latest_result = processed_result + if after_processed_commit is not None: + after_processed_commit(pending, processed_result) + return latest_result diff --git a/policyengine_api/services/v2/__init__.py b/policyengine_api/services/v2/__init__.py new file mode 100644 index 000000000..7452ae05a --- /dev/null +++ b/policyengine_api/services/v2/__init__.py @@ -0,0 +1 @@ +"""Application services for native API v2 resources.""" diff --git a/policyengine_api/services/v2/metadata/__init__.py b/policyengine_api/services/v2/metadata/__init__.py new file mode 100644 index 000000000..fcf76cce7 --- /dev/null +++ b/policyengine_api/services/v2/metadata/__init__.py @@ -0,0 +1 @@ +"""Application services for API v2 metadata reads.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/__init__.py b/policyengine_api/services/v2/metadata/database_connectors/__init__.py new file mode 100644 index 000000000..e1d0feec0 --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for v2 metadata services.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads.py b/policyengine_api/services/v2/metadata/database_connectors/reads.py new file mode 100644 index 000000000..d44aaa432 --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads.py @@ -0,0 +1,38 @@ +"""Shared database selections for v2 metadata operations.""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy.exc import SQLAlchemyError +from sqlmodel import Session + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + SelectedCatalog, + select_catalog, +) + + +def read_metadata_catalog( + session: Session, + *, + country_id: str, + running_policyengine_version: str, + policyengine_version: str | None, +) -> SelectedCatalog: + return select_catalog( + session, + country_id=country_id, + running_policyengine_version=running_policyengine_version, + policyengine_version=policyengine_version, + ) + + +def read_rows(session: Session, statement: Any) -> list[Any]: + try: + return list(session.exec(statement).all()) + except SQLAlchemyError as error: + raise MetadataCatalogUnavailableError( + "the v2 metadata catalog cannot be queried" + ) from error diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py new file mode 100644 index 000000000..d719e443a --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py @@ -0,0 +1,42 @@ +"""Database selections for logical input-dataset metadata.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Dataset +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + + +def read_datasets( + session: Session, *, model_version_id: UUID, offset: int, limit: int +) -> list[Dataset]: + return read_rows( + session, + select(Dataset) + .where( + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ) + .order_by(col(Dataset.name)) + .offset(offset) + .limit(limit + 1), + ) + + +def read_dataset( + session: Session, *, model_version_id: UUID, dataset_id: UUID +) -> Dataset | None: + rows = read_rows( + session, + select(Dataset).where( + col(Dataset.id) == dataset_id, + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ), + ) + return rows[0] if rows else None diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py new file mode 100644 index 000000000..011f2d75d --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py @@ -0,0 +1,150 @@ +"""Database selections for direct parameter-tree children.""" + +from __future__ import annotations + +from typing import Any +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Session, col + +from policyengine_api.data.v2.models import Parameter, ParameterNode +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + + +def _escaped_like(value: str) -> str: + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +def _path_segment(remainder: Any, dialect: str) -> Any: + dot_position = ( + sa.func.instr(remainder, ".") + if dialect == "sqlite" + else sa.func.strpos(remainder, ".") + ) + return sa.case( + (dot_position > 0, sa.func.substr(remainder, 1, dot_position - 1)), + else_=remainder, + ) + + +def _child_path(column: Any, prefix: str, dialect: str) -> Any: + remainder = sa.func.substr(column, len(prefix) + 1) + return sa.literal(prefix) + _path_segment(remainder, dialect) + + +def _direct_child_path(column: Any, parent_path: Any, dialect: str) -> Any: + remainder = sa.func.substr(column, sa.func.length(parent_path) + 2) + return parent_path + "." + _path_segment(remainder, dialect) + + +def _has_path_prefix(column: Any, parent_path: Any) -> Any: + return ( + sa.func.substr(column, 1, sa.func.length(parent_path) + 1) == parent_path + "." + ) + + +def _parameter_children_statement( + *, + model_version_id: UUID, + parent_path: str, + dialect: str, + offset: int, + limit: int, +) -> Any: + prefix = f"{parent_path}." if parent_path else "" + escaped_prefix = _escaped_like(prefix) + node_name = col(ParameterNode.name) + node_model_version_id = col(ParameterNode.tax_benefit_model_version_id) + parameter_name = col(Parameter.name) + parameter_model_version_id = col(Parameter.tax_benefit_model_version_id) + node_child_path = _child_path(node_name, prefix, dialect) + parameter_child_path = _child_path(parameter_name, prefix, dialect) + paths = sa.union( + sa.select(node_child_path.label("path")).where( + node_model_version_id == model_version_id, + node_name.like(f"{escaped_prefix}%", escape="\\"), + ), + sa.select(parameter_child_path.label("path")).where( + parameter_model_version_id == model_version_id, + parameter_name.like(f"{escaped_prefix}%", escape="\\"), + ), + ).subquery() + direct_child_paths = sa.union( + sa.select(_direct_child_path(node_name, paths.c.path, dialect).label("path")) + .where( + node_model_version_id == model_version_id, + _has_path_prefix(node_name, paths.c.path), + ) + .correlate(paths), + sa.select( + _direct_child_path(parameter_name, paths.c.path, dialect).label("path") + ) + .where( + parameter_model_version_id == model_version_id, + _has_path_prefix(parameter_name, paths.c.path), + ) + .correlate(paths), + ).subquery() + direct_child_count = ( + sa.select(sa.func.count()) + .select_from(direct_child_paths) + .correlate(paths) + .scalar_subquery() + ) + parameter_id = col(Parameter.id) + is_node = sa.or_(direct_child_count > 0, parameter_id.is_(None)) + return ( + sa.select( + paths.c.path, + sa.func.coalesce(col(ParameterNode.label), col(Parameter.label)).label( + "label" + ), + sa.case((is_node, "node"), else_="parameter").label("type"), + sa.case((is_node, direct_child_count), else_=None).label("child_count"), + parameter_id.label("parameter_id"), + col(Parameter.label).label("parameter_label"), + col(Parameter.description).label("parameter_description"), + col(Parameter.data_type).label("parameter_data_type"), + col(Parameter.unit).label("parameter_unit"), + ) + .select_from( + paths.outerjoin( + ParameterNode, + sa.and_( + node_name == paths.c.path, + node_model_version_id == model_version_id, + ), + ).outerjoin( + Parameter, + sa.and_( + parameter_name == paths.c.path, + parameter_model_version_id == model_version_id, + ), + ) + ) + .order_by(paths.c.path) + .offset(offset) + .limit(limit + 1) + ) + + +def read_parameter_children( + session: Session, + *, + model_version_id: UUID, + parent_path: str, + dialect: str, + offset: int, + limit: int, +) -> list[Any]: + return read_rows( + session, + _parameter_children_statement( + model_version_id=model_version_id, + parent_path=parent_path, + dialect=dialect, + offset=offset, + limit=limit, + ), + ) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py new file mode 100644 index 000000000..69817fb5c --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -0,0 +1,112 @@ +"""Database selections for parameter and canonical-value metadata.""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Parameter, ParameterValue +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + + +def _escape_like(value: str) -> str: + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +def read_parameters( + session: Session, + *, + model_version_id: UUID, + offset: int, + limit: int, + search: str | None, +) -> list[Parameter]: + statement = select(Parameter).where( + col(Parameter.tax_benefit_model_version_id) == model_version_id + ) + if search: + pattern = f"%{_escape_like(search)}%" + statement = statement.where( + sa.or_( + col(Parameter.name).ilike(pattern, escape="\\"), + col(Parameter.label).ilike(pattern, escape="\\"), + col(Parameter.description).ilike(pattern, escape="\\"), + ) + ) + return read_rows( + session, + statement.order_by(col(Parameter.name)).offset(offset).limit(limit + 1), + ) + + +def read_parameter( + session: Session, *, model_version_id: UUID, parameter_id: UUID +) -> Parameter | None: + rows = read_rows( + session, + select(Parameter).where( + col(Parameter.id) == parameter_id, + col(Parameter.tax_benefit_model_version_id) == model_version_id, + ), + ) + return rows[0] if rows else None + + +def read_parameter_values( + session: Session, + *, + model_version_id: UUID, + parameter_id: UUID | None, + selected_day: datetime | None, + offset: int, + limit: int, +) -> list[ParameterValue]: + statement = ( + select(ParameterValue) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), + ) + ) + if parameter_id is not None: + statement = statement.where(col(ParameterValue.parameter_id) == parameter_id) + if selected_day is not None: + statement = statement.where( + col(ParameterValue.start_date) <= selected_day, + sa.or_( + col(ParameterValue.end_date).is_(None), + col(ParameterValue.end_date) >= selected_day, + ), + ) + return read_rows( + session, + statement.order_by( + col(Parameter.name), + col(ParameterValue.start_date).desc(), + col(ParameterValue.id), + ) + .offset(offset) + .limit(limit + 1), + ) + + +def read_parameter_value( + session: Session, *, model_version_id: UUID, value_id: UUID +) -> ParameterValue | None: + rows = read_rows( + session, + select(ParameterValue) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where( + col(ParameterValue.id) == value_id, + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), + ), + ) + return rows[0] if rows else None diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_regions.py b/policyengine_api/services/v2/metadata/database_connectors/reads_regions.py new file mode 100644 index 000000000..84f85d668 --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_regions.py @@ -0,0 +1,64 @@ +"""Database selections for region and economy-option metadata.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Dataset, Region +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + + +def read_regions( + session: Session, + *, + model_version_id: UUID, + region_type: str | None = None, + offset: int | None = None, + limit: int | None = None, +) -> list[Region]: + statement = select(Region).where( + col(Region.tax_benefit_model_version_id) == model_version_id + ) + if region_type is not None: + statement = statement.where(col(Region.region_type) == region_type) + statement = statement.order_by(col(Region.code)) + if offset is not None: + statement = statement.offset(offset) + if limit is not None: + statement = statement.limit(limit + 1) + return read_rows(session, statement) + + +def read_region( + session: Session, + *, + model_version_id: UUID, + region_id: UUID | None = None, + region_code: str | None = None, +) -> Region | None: + statement = select(Region).where( + col(Region.tax_benefit_model_version_id) == model_version_id + ) + if region_id is not None: + statement = statement.where(col(Region.id) == region_id) + if region_code is not None: + statement = statement.where(col(Region.code) == region_code) + rows = read_rows(session, statement) + return rows[0] if rows else None + + +def read_input_dataset( + session: Session, *, model_version_id: UUID, dataset_id: UUID +) -> Dataset | None: + rows = read_rows( + session, + select(Dataset).where( + col(Dataset.id) == dataset_id, + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ), + ) + return rows[0] if rows else None diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py new file mode 100644 index 000000000..f9ba8dad9 --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py @@ -0,0 +1,54 @@ +"""Database selections for variable metadata.""" + +from __future__ import annotations + +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Variable +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + + +def _escape_like(value: str) -> str: + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +def read_variables( + session: Session, + *, + model_version_id: UUID, + offset: int, + limit: int, + search: str | None, +) -> list[Variable]: + statement = select(Variable).where( + col(Variable.tax_benefit_model_version_id) == model_version_id + ) + if search: + pattern = f"%{_escape_like(search)}%" + statement = statement.where( + sa.or_( + col(Variable.name).ilike(pattern, escape="\\"), + col(Variable.label).ilike(pattern, escape="\\"), + col(Variable.description).ilike(pattern, escape="\\"), + ) + ) + return read_rows( + session, + statement.order_by(col(Variable.name)).offset(offset).limit(limit + 1), + ) + + +def read_variable( + session: Session, *, model_version_id: UUID, variable_id: UUID +) -> Variable | None: + rows = read_rows( + session, + select(Variable).where( + col(Variable.id) == variable_id, + col(Variable.tax_benefit_model_version_id) == model_version_id, + ), + ) + return rows[0] if rows else None diff --git a/policyengine_api/services/v2/metadata/database_session.py b/policyengine_api/services/v2/metadata/database_session.py new file mode 100644 index 000000000..32c6f0d0d --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_session.py @@ -0,0 +1,23 @@ +"""Database session lifetime management for v2 metadata services.""" + +from __future__ import annotations + +from sqlmodel import Session + + +class MetadataDatabaseSession: + """Own the request-scoped read session used by metadata services.""" + + def __init__(self, session: Session) -> None: + self._session = session + + @property + def session(self) -> Session: + return self._session + + @property + def dialect_name(self) -> str: + return self._session.get_bind().dialect.name + + def close(self) -> None: + self._session.close() diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py new file mode 100644 index 000000000..1e24861f5 --- /dev/null +++ b/policyengine_api/services/v2/metadata/services.py @@ -0,0 +1,491 @@ +"""Router-facing services for API v2 metadata reads.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import UUID + +from policyengine_api.data.v2.catalog.catalog_selection import ( + SelectedCatalog, + validate_policyengine_version, +) +from policyengine_api.services.v2.metadata.database_connectors.reads import ( + read_metadata_catalog, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_datasets import ( + read_dataset, + read_datasets, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_tree import ( + read_parameter_children, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( + read_parameter, + read_parameter_value, + read_parameter_values, + read_parameters, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_regions import ( + read_input_dataset, + read_region, + read_regions, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_variables import ( + read_variable, + read_variables, +) +from policyengine_api.services.v2.metadata.database_session import ( + MetadataDatabaseSession, +) +from policyengine_api.services.v2.metadata.transformations import ( + metadata_dataset, + metadata_economy_options, + metadata_model, + metadata_model_selection, + metadata_model_version, + metadata_parameter, + metadata_parameter_children, + metadata_parameter_value, + metadata_region, + metadata_variable, + page_result, + utc_day_start, +) +from policyengine_api.services.v2.metadata.types import ( + MetadataCanonicalParameterValue, + MetadataDataset, + MetadataDetailResult, + MetadataEconomyOptionsResult, + MetadataModel, + MetadataModelSelectionResult, + MetadataModelVersionDetail, + MetadataPageResult, + MetadataParameterChild, + MetadataParameterSummary, + MetadataRegion, + MetadataVariable, +) +from policyengine_api.services.v2.metadata.validators import ( + require_metadata_resource, + validate_economy_options, + validate_metadata_page, +) + + +class V2MetadataService: + """Sequence metadata reads through a request-scoped database session.""" + + def __init__( + self, + database_session: MetadataDatabaseSession, + *, + running_policyengine_version: str, + ) -> None: + self._database_session = database_session + self._running_policyengine_version = validate_policyengine_version( + running_policyengine_version + ) + + def close(self) -> None: + self._database_session.close() + + def _select_catalog( + self, country_id: str, policyengine_version: str | None + ) -> SelectedCatalog: + return read_metadata_catalog( + self._database_session.session, + country_id=country_id, + running_policyengine_version=self._running_policyengine_version, + policyengine_version=policyengine_version, + ) + + def _select_paginated_catalog( + self, + country_id: str, + policyengine_version: str | None, + *, + offset: int, + limit: int, + ) -> SelectedCatalog: + validate_metadata_page(offset, limit) + return self._select_catalog(country_id, policyengine_version) + + def list_models( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataModel]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = [metadata_model(selected)] if offset == 0 else [] + return page_result(selected, rows, offset=offset, limit=limit) + + def get_model( + self, + country_id: str, + model_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataModel]: + selected = self._select_catalog(country_id, policyengine_version) + item = require_metadata_resource( + metadata_model(selected) if selected.model.id == model_id else None, + description=f"model {model_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, item=item + ) + + def get_model_by_country( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataModelSelectionResult: + return metadata_model_selection( + self._select_catalog(country_id, policyengine_version) + ) + + def list_model_versions( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataModelVersionDetail]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = [metadata_model_version(selected)] if offset == 0 else [] + return page_result(selected, rows, offset=offset, limit=limit) + + def get_model_version( + self, + country_id: str, + version_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataModelVersionDetail]: + selected = self._select_catalog(country_id, policyengine_version) + item = require_metadata_resource( + ( + metadata_model_version(selected) + if selected.model_version.id == version_id + else None + ), + description=f"model version {version_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, item=item + ) + + def list_variables( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + search: str | None = None, + ) -> MetadataPageResult[MetadataVariable]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_variables( + self._database_session.session, + model_version_id=selected.model_version.id, + offset=offset, + limit=limit, + search=search, + ) + return page_result( + selected, + [metadata_variable(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_variable( + self, + country_id: str, + variable_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataVariable]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_variable( + self._database_session.session, + model_version_id=selected.model_version.id, + variable_id=variable_id, + ), + description=f"variable {variable_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_variable(row), + ) + + def list_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + search: str | None = None, + ) -> MetadataPageResult[MetadataParameterSummary]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_parameters( + self._database_session.session, + model_version_id=selected.model_version.id, + offset=offset, + limit=limit, + search=search, + ) + return page_result( + selected, + [metadata_parameter(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_parameter( + self, + country_id: str, + parameter_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataParameterSummary]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_parameter( + self._database_session.session, + model_version_id=selected.model_version.id, + parameter_id=parameter_id, + ), + description=f"parameter {parameter_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_parameter(row), + ) + + def list_parameter_children( + self, + country_id: str, + policyengine_version: str | None = None, + *, + parent_path: str = "", + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataParameterChild]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_parameter_children( + self._database_session.session, + model_version_id=selected.model_version.id, + parent_path=parent_path, + dialect=self._database_session.dialect_name, + offset=offset, + limit=limit, + ) + return page_result( + selected, + metadata_parameter_children(rows), + offset=offset, + limit=limit, + ) + + def list_parameter_values( + self, + country_id: str, + policyengine_version: str | None = None, + *, + parameter_id: UUID | None = None, + current: bool = False, + offset: int = 0, + limit: int = 100, + now: datetime | None = None, + ) -> MetadataPageResult[MetadataCanonicalParameterValue]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + selected_day = ( + utc_day_start(now or datetime.now(timezone.utc)) if current else None + ) + rows = read_parameter_values( + self._database_session.session, + model_version_id=selected.model_version.id, + parameter_id=parameter_id, + selected_day=selected_day, + offset=offset, + limit=limit, + ) + return page_result( + selected, + [metadata_parameter_value(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_parameter_value( + self, + country_id: str, + value_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataCanonicalParameterValue]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_parameter_value( + self._database_session.session, + model_version_id=selected.model_version.id, + value_id=value_id, + ), + description=f"parameter value {value_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_parameter_value(row), + ) + + def list_datasets( + self, + country_id: str, + policyengine_version: str | None = None, + *, + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataDataset]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_datasets( + self._database_session.session, + model_version_id=selected.model_version.id, + offset=offset, + limit=limit, + ) + return page_result( + selected, + [metadata_dataset(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_dataset( + self, + country_id: str, + dataset_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataDataset]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_dataset( + self._database_session.session, + model_version_id=selected.model_version.id, + dataset_id=dataset_id, + ), + description=f"dataset {dataset_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_dataset(row), + ) + + def list_regions( + self, + country_id: str, + policyengine_version: str | None = None, + *, + region_type: str | None = None, + offset: int = 0, + limit: int = 100, + ) -> MetadataPageResult[MetadataRegion]: + selected = self._select_paginated_catalog( + country_id, policyengine_version, offset=offset, limit=limit + ) + rows = read_regions( + self._database_session.session, + model_version_id=selected.model_version.id, + region_type=region_type, + offset=offset, + limit=limit, + ) + return page_result( + selected, + [metadata_region(row) for row in rows], + offset=offset, + limit=limit, + ) + + def get_region( + self, + country_id: str, + region_id: UUID, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataRegion]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_region( + self._database_session.session, + model_version_id=selected.model_version.id, + region_id=region_id, + ), + description=f"region {region_id}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_region(row), + ) + + def get_region_by_code( + self, + country_id: str, + region_code: str, + policyengine_version: str | None = None, + ) -> MetadataDetailResult[MetadataRegion]: + selected = self._select_catalog(country_id, policyengine_version) + row = require_metadata_resource( + read_region( + self._database_session.session, + model_version_id=selected.model_version.id, + region_code=region_code, + ), + description=f"region {region_code!r}", + ) + return MetadataDetailResult( + policyengine_version=selected.policyengine_version, + item=metadata_region(row), + ) + + def get_economy_options( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataEconomyOptionsResult: + selected = self._select_catalog(country_id, policyengine_version) + regions = read_regions( + self._database_session.session, + model_version_id=selected.model_version.id, + ) + national_region = next( + (region for region in regions if region.code == country_id), None + ) + national_dataset = ( + read_input_dataset( + self._database_session.session, + model_version_id=selected.model_version.id, + dataset_id=national_region.default_dataset_id, + ) + if national_region is not None + else None + ) + dataset = validate_economy_options( + selected, + country_id=country_id, + regions=regions, + national_dataset=national_dataset, + ) + return metadata_economy_options( + selected, regions=regions, national_dataset=dataset + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py new file mode 100644 index 000000000..83208f41f --- /dev/null +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -0,0 +1,201 @@ +"""Pure representation transformations for v2 metadata operations.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, TypeVar + +from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog +from policyengine_api.data.v2.models import ( + Dataset, + Parameter, + ParameterValue, + Region, + Variable, +) +from policyengine_api.dataset_display import get_dataset_display_label +from policyengine_api.services.v2.metadata.types import ( + MetadataCanonicalParameterValue, + MetadataDataset, + MetadataDatasetOption, + MetadataEconomyOptionsResult, + MetadataModel, + MetadataModelSelectionResult, + MetadataModelVersionDetail, + MetadataPageResult, + MetadataParameterChild, + MetadataParameterSummary, + MetadataRegion, + MetadataRegionOption, + MetadataRegionType, + MetadataTimePeriodOption, + MetadataVariable, +) + + +ResourceT = TypeVar("ResourceT") + + +def page_result( + selected: SelectedCatalog, + rows: list[ResourceT], + *, + offset: int, + limit: int, +) -> MetadataPageResult[ResourceT]: + return MetadataPageResult( + policyengine_version=selected.policyengine_version, + items=rows[:limit], + offset=offset, + limit=limit, + has_more=len(rows) > limit, + ) + + +def metadata_model(selected: SelectedCatalog) -> MetadataModel: + return MetadataModel( + id=selected.model.id, + name=selected.model.name, + description=selected.model_version.description, + ) + + +def metadata_model_version(selected: SelectedCatalog) -> MetadataModelVersionDetail: + return MetadataModelVersionDetail( + id=selected.model_version.id, + model_id=selected.model.id, + version=selected.model_version.version, + description=selected.model_version.description, + current_law_id=selected.model_version.current_law_id, + metadata_time_periods=selected.model_version.metadata_time_periods, + ) + + +def metadata_model_selection(selected: SelectedCatalog) -> MetadataModelSelectionResult: + return MetadataModelSelectionResult( + policyengine_version=selected.policyengine_version, + model=metadata_model(selected), + model_version=metadata_model_version(selected), + ) + + +def metadata_variable(variable: Variable) -> MetadataVariable: + return MetadataVariable( + id=variable.id, + name=variable.name, + label=variable.label, + entity=variable.entity, + description=variable.description, + data_type=variable.data_type, + possible_values=variable.possible_values, + default_value=variable.default_value, + adds=variable.adds, + subtracts=variable.subtracts, + ) + + +def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: + return MetadataParameterSummary( + id=parameter.id, + name=parameter.name, + label=parameter.label, + description=parameter.description, + data_type=parameter.data_type, + unit=parameter.unit, + ) + + +def metadata_parameter_value(value: ParameterValue) -> MetadataCanonicalParameterValue: + return MetadataCanonicalParameterValue( + id=value.id, + parameter_id=value.parameter_id, + value=value.value_json, + start_date=value.start_date, + end_date=value.end_date, + ) + + +def utc_day_start(selected_time: datetime) -> datetime: + if selected_time.tzinfo is None: + selected_time = selected_time.replace(tzinfo=timezone.utc) + return selected_time.astimezone(timezone.utc).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + + +def metadata_dataset(dataset: Dataset) -> MetadataDataset: + return MetadataDataset( + id=dataset.id, + name=dataset.name, + description=dataset.description, + year=dataset.year, + ) + + +def metadata_region(region: Region) -> MetadataRegion: + return MetadataRegion( + id=region.id, + code=region.code, + label=region.label, + region_type=MetadataRegionType(region.region_type.value), + requires_filter=region.requires_filter, + filter_field=region.filter_field, + filter_value=region.filter_value, + filter_strategy=region.filter_strategy, + parent_code=region.parent_code, + state_code=region.state_code, + state_name=region.state_name, + default_dataset_id=region.default_dataset_id, + ) + + +def metadata_parameter_children(rows: list[Any]) -> list[MetadataParameterChild]: + items = [] + for row in rows: + parameter = None + if row.type == "parameter": + parameter = MetadataParameterSummary( + id=row.parameter_id, + name=row.path, + label=row.parameter_label, + description=row.parameter_description, + data_type=row.parameter_data_type, + unit=row.parameter_unit, + ) + items.append( + MetadataParameterChild( + path=row.path, + label=row.label or row.path.rsplit(".", 1)[-1], + type=row.type, + child_count=row.child_count, + parameter=parameter, + ) + ) + return items + + +def metadata_economy_options( + selected: SelectedCatalog, *, regions: list[Region], national_dataset: Dataset +) -> MetadataEconomyOptionsResult: + return MetadataEconomyOptionsResult( + policyengine_version=selected.policyengine_version, + current_law_id=selected.model_version.current_law_id, + region=[ + MetadataRegionOption( + name=region.code, + label=region.label, + type=MetadataRegionType(region.region_type.value), + ) + for region in regions + ], + time_period=[ + MetadataTimePeriodOption(name=year, label=str(year)) + for year in selected.model_version.metadata_time_periods + ], + datasets=[ + MetadataDatasetOption( + name=national_dataset.name, + label=get_dataset_display_label(national_dataset.name), + ) + ], + ) diff --git a/policyengine_api/data/v2/catalog/schemas.py b/policyengine_api/services/v2/metadata/types.py similarity index 54% rename from policyengine_api/data/v2/catalog/schemas.py rename to policyengine_api/services/v2/metadata/types.py index ddd01fb8e..fef5b7894 100644 --- a/policyengine_api/data/v2/catalog/schemas.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -1,13 +1,13 @@ -"""Typed response schemas for the dormant v2 metadata preview.""" +"""Framework-independent data exchanged by v2 metadata layers.""" from __future__ import annotations from datetime import datetime from enum import StrEnum -from typing import Annotated, Generic, Literal, TypeVar +from typing import Generic, Literal, TypeVar from uuid import UUID -from pydantic import BaseModel, ConfigDict, JsonValue, StringConstraints +from pydantic import BaseModel, ConfigDict, JsonValue class StrictResponseModel(BaseModel): @@ -150,118 +150,3 @@ class MetadataEconomyOptionsResult(StrictResponseModel): region: list[MetadataRegionOption] time_period: list[MetadataTimePeriodOption] datasets: list[MetadataDatasetOption] - - -class MetadataResourceSuccessResponse(StrictResponseModel, Generic[ResourceT]): - status: Literal["ok"] = "ok" - message: None = None - result: ResourceT - - -class MetadataModelPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataModel]] -): - pass - - -class MetadataModelDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataModel]] -): - pass - - -class MetadataModelSelectionResponse( - MetadataResourceSuccessResponse[MetadataModelSelectionResult] -): - pass - - -class MetadataModelVersionPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataModelVersionDetail]] -): - pass - - -class MetadataModelVersionDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataModelVersionDetail]] -): - pass - - -class MetadataVariablePageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataVariable]] -): - pass - - -class MetadataVariableDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataVariable]] -): - pass - - -class MetadataParameterPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataParameterSummary]] -): - pass - - -class MetadataParameterDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataParameterSummary]] -): - pass - - -class MetadataParameterChildPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataParameterChild]] -): - pass - - -class MetadataParameterValuePageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataCanonicalParameterValue]] -): - pass - - -class MetadataParameterValueDetailResponse( - MetadataResourceSuccessResponse[ - MetadataDetailResult[MetadataCanonicalParameterValue] - ] -): - pass - - -class MetadataDatasetPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataDataset]] -): - pass - - -class MetadataDatasetDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataDataset]] -): - pass - - -class MetadataRegionPageResponse( - MetadataResourceSuccessResponse[MetadataPageResult[MetadataRegion]] -): - pass - - -class MetadataRegionDetailResponse( - MetadataResourceSuccessResponse[MetadataDetailResult[MetadataRegion]] -): - pass - - -class MetadataEconomyOptionsResponse( - MetadataResourceSuccessResponse[MetadataEconomyOptionsResult] -): - pass - - -class MetadataErrorResponse(StrictResponseModel): - status: Literal["error"] = "error" - message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] diff --git a/policyengine_api/services/v2/metadata/validators.py b/policyengine_api/services/v2/metadata/validators.py new file mode 100644 index 000000000..1ca60c3a7 --- /dev/null +++ b/policyengine_api/services/v2/metadata/validators.py @@ -0,0 +1,70 @@ +"""Database-independent validation for v2 metadata operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, TypeVar + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, +) + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog + from policyengine_api.data.v2.models import Dataset, Region + + +class MetadataResourceNotFoundError(LookupError): + """Raised when a selected catalog does not contain a requested resource.""" + + +class InvalidMetadataPageError(ValueError): + """Raised when collection pagination is outside the documented bounds.""" + + +def validate_metadata_page(offset: int, limit: int) -> tuple[int, int]: + if offset < 0: + raise InvalidMetadataPageError("offset must be at least 0") + if not 1 <= limit <= 500: + raise InvalidMetadataPageError("limit must be between 1 and 500") + return offset, limit + + +ResourceT = TypeVar("ResourceT") + + +def require_metadata_resource( + resource: ResourceT | None, *, description: str +) -> ResourceT: + if resource is None: + raise MetadataResourceNotFoundError(f"{description} was not found") + return resource + + +def validate_economy_options( + selected: "SelectedCatalog", + *, + country_id: str, + regions: list["Region"], + national_dataset: "Dataset | None", +) -> "Dataset": + """Validate preloaded records required by the economy-options response.""" + + if not any(region.code == country_id for region in regions): + raise MetadataCatalogUnavailableError( + f"the {country_id} national v2 region is absent" + ) + if national_dataset is None: + raise MetadataCatalogUnavailableError( + f"the {country_id} national v2 dataset is absent" + ) + time_periods = selected.model_version.metadata_time_periods + if ( + not isinstance(selected.model_version.current_law_id, int) + or not isinstance(time_periods, list) + or not time_periods + or any(not isinstance(year, int) for year in time_periods) + ): + raise MetadataCatalogUnavailableError( + f"the {country_id} v2 model-version options are incomplete" + ) + return national_dataset diff --git a/policyengine_api/services/v2/policies/__init__.py b/policyengine_api/services/v2/policies/__init__.py new file mode 100644 index 000000000..391a86685 --- /dev/null +++ b/policyengine_api/services/v2/policies/__init__.py @@ -0,0 +1 @@ +"""Application services for immutable v2 policies.""" diff --git a/policyengine_api/services/v2/policies/database_connectors/__init__.py b/policyengine_api/services/v2/policies/database_connectors/__init__.py new file mode 100644 index 000000000..4cb277126 --- /dev/null +++ b/policyengine_api/services/v2/policies/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for v2 policy services.""" diff --git a/policyengine_api/services/v2/policies/database_connectors/creates.py b/policyengine_api/services/v2/policies/database_connectors/creates.py new file mode 100644 index 000000000..1821921b4 --- /dev/null +++ b/policyengine_api/services/v2/policies/database_connectors/creates.py @@ -0,0 +1,78 @@ +"""Database inserts used by immutable v2 policy operations.""" + +from __future__ import annotations + +from uuid import UUID, uuid4 + +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, col + +from policyengine_api.data.v2.models import LegacyPolicyMapping, ParameterValue, Policy +from policyengine_api.services.v2.policies.types import ResolvedPolicyCreationInput + + +def create_policy( + session: Session, + policy_input: ResolvedPolicyCreationInput, + *, + canonicalization_version: int, + content_hash: str, +) -> UUID | None: + policy_id = uuid4() + statement = ( + insert(Policy) + .values( + id=policy_id, + country_id=policy_input.country_id, + tax_benefit_model_id=policy_input.tax_benefit_model_id, + tax_benefit_model_version_id=policy_input.tax_benefit_model_version_id, + canonicalization_version=canonicalization_version, + content_hash=content_hash, + ) + .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") + .returning(col(Policy.id)) + ) + return session.execute(statement).scalar_one_or_none() + + +def create_parameter_values( + session: Session, + *, + policy_id: UUID, + policy_input: ResolvedPolicyCreationInput, +) -> None: + session.add_all( + [ + ParameterValue( + policy_id=policy_id, + dynamic_id=None, + parameter_id=value.parameter_id, + value_json=value.value, + start_date=value.start_date, + end_date=value.end_date, + ) + for value in policy_input.parameter_values + ] + ) + session.flush() + + +def create_legacy_policy_mapping( + session: Session, + *, + country_id: str, + legacy_policy_id: int, + source_policy_hash: str, + policy_id: UUID, +) -> UUID | None: + return session.execute( + insert(LegacyPolicyMapping) + .values( + country_id=country_id, + legacy_policy_id=legacy_policy_id, + policy_id=policy_id, + source_policy_hash=source_policy_hash, + ) + .on_conflict_do_nothing(constraint="uq_legacy_policy_mappings_country_legacy") + .returning(col(LegacyPolicyMapping.id)) + ).scalar_one_or_none() diff --git a/policyengine_api/services/v2/policies/database_connectors/reads.py b/policyengine_api/services/v2/policies/database_connectors/reads.py new file mode 100644 index 000000000..0276a1b43 --- /dev/null +++ b/policyengine_api/services/v2/policies/database_connectors/reads.py @@ -0,0 +1,148 @@ +"""Database selections used by immutable v2 policy operations.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, col, select + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v2.catalog.catalog_selection import ( + SelectedCatalog, + select_catalog, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModelVersion, +) + + +def read_policy_catalog( + session: Session, + country_id: str, + *, + policyengine_version: str | None = None, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> SelectedCatalog: + return select_catalog( + session, + country_id=country_id, + running_policyengine_version=running_policyengine_version, + policyengine_version=policyengine_version, + ) + + +def read_version_parameter_ids( + session: Session, *, model_version_id: UUID, requested_ids: set[UUID] +) -> set[UUID]: + if not requested_ids: + return set() + return set( + session.exec( + select(Parameter.id).where( + Parameter.tax_benefit_model_version_id == model_version_id, + col(Parameter.id).in_(requested_ids), + ) + ).all() + ) + + +def read_parameters_by_name( + session: Session, *, model_version_id: UUID, names: set[str] +) -> dict[str, Parameter]: + if not names: + return {} + parameters = session.exec( + select(Parameter).where( + Parameter.tax_benefit_model_version_id == model_version_id, + col(Parameter.name).in_(names), + ) + ).all() + return {parameter.name: parameter for parameter in parameters} + + +def read_policy_by_content_identity( + session: Session, *, canonicalization_version: int, content_hash: str +) -> Policy | None: + return session.exec( + select(Policy).where( + Policy.canonicalization_version == canonicalization_version, + Policy.content_hash == content_hash, + ) + ).one_or_none() + + +def read_model_version( + session: Session, model_version_id: UUID +) -> TaxBenefitModelVersion | None: + return session.get(TaxBenefitModelVersion, model_version_id) + + +def read_parameter_values(session: Session, policy_id: UUID) -> list[ParameterValue]: + return list( + session.exec( + select(ParameterValue).where(ParameterValue.policy_id == policy_id) + ).all() + ) + + +def read_legacy_policy_mapping( + session: Session, *, country_id: str, legacy_policy_id: int, lock: bool +) -> LegacyPolicyMapping | None: + statement = select(LegacyPolicyMapping).where( + LegacyPolicyMapping.country_id == country_id, + LegacyPolicyMapping.legacy_policy_id == legacy_policy_id, + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def read_policy_row( + session: Session, *, country_id: str, policy_id: UUID +) -> Policy | None: + return session.exec( + select(Policy).where(Policy.id == policy_id, Policy.country_id == country_id) + ).one_or_none() + + +def read_policy_rows( + session: Session, + *, + country_id: str, + tax_benefit_model_id: UUID | None, + offset: int, + limit: int, +) -> list[Policy]: + statement = select(Policy).where(Policy.country_id == country_id) + if tax_benefit_model_id is not None: + statement = statement.where(Policy.tax_benefit_model_id == tax_benefit_model_id) + return list( + session.exec( + statement.order_by(col(Policy.created_at), col(Policy.id)) + .offset(offset) + .limit(limit + 1) + ).all() + ) + + +def read_parameter_values_with_names( + session: Session, policy_ids: list[UUID] +) -> list[tuple[ParameterValue, str]]: + if not policy_ids: + return [] + return list( + session.exec( + select(ParameterValue, Parameter.name) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where(col(ParameterValue.policy_id).in_(policy_ids)) + .order_by( + col(Parameter.name), + col(ParameterValue.start_date), + col(ParameterValue.id), + ) + ).all() + ) diff --git a/policyengine_api/services/v2/policies/database_session.py b/policyengine_api/services/v2/policies/database_session.py new file mode 100644 index 000000000..3a8a6e057 --- /dev/null +++ b/policyengine_api/services/v2/policies/database_session.py @@ -0,0 +1,26 @@ +"""Database session lifetime management for v2 policy services.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + + +class PolicyDatabaseSession: + """Open read sessions and transaction-scoped sessions for policy services.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + with self._session_factory() as session: + yield session + + @contextmanager + def transaction(self) -> Iterator[Session]: + with self._session_factory.begin() as session: + yield session diff --git a/policyengine_api/services/v2/policies/services.py b/policyengine_api/services/v2/policies/services.py new file mode 100644 index 000000000..6f5ac9959 --- /dev/null +++ b/policyengine_api/services/v2/policies/services.py @@ -0,0 +1,320 @@ +"""Router-facing and legacy-mirroring services for v2 policies.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from uuid import UUID + +from sqlmodel import Session + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION +from policyengine_api.services.v2.policies.database_connectors.creates import ( + create_legacy_policy_mapping, + create_parameter_values, + create_policy, +) +from policyengine_api.services.v2.policies.database_connectors.reads import ( + read_legacy_policy_mapping, + read_model_version, + read_parameter_values, + read_parameter_values_with_names, + read_parameters_by_name, + read_policy_by_content_identity, + read_policy_catalog, + read_policy_row, + read_policy_rows, + read_version_parameter_ids, +) +from policyengine_api.services.v2.policies.database_session import PolicyDatabaseSession +from policyengine_api.services.v2.policies.transformations import ( + canonical_policy_document, + canonicalize_policy, + policy_page, + policy_parameter_values_by_policy, + policy_read, + stored_policy_creation_input, + translate_legacy_policy, +) +from policyengine_api.services.v2.policies.types import ( + CanonicalPolicyContent, + LegacyPolicyPersistenceResult, + LegacyPolicySnapshot, + NativePolicyCreation, + NativePolicyCreationInput, + PolicyCreationInput, + PolicyCreationResult, + PolicyPage, + PolicyRead, + ResolvedPolicyCreationInput, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, + PolicyContentHashCollisionError, + PolicyCreationIntegrityError, + PolicyNotFoundError, + validate_policy_catalog, + verify_legacy_policy_mapping, +) + + +def resolve_policy_creation_input( + session: Session, + policy_input: PolicyCreationInput, + *, + policyengine_version: str | None = None, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> ResolvedPolicyCreationInput: + selected = read_policy_catalog( + session, + policy_input.country_id, + policyengine_version=policyengine_version, + running_policyengine_version=running_policyengine_version, + ) + requested_parameter_ids = { + value.parameter_id for value in policy_input.parameter_values + } + resolved_parameter_ids = read_version_parameter_ids( + session, + model_version_id=selected.model_version.id, + requested_ids=requested_parameter_ids, + ) + return validate_policy_catalog( + policy_input, + selected=selected, + resolved_parameter_ids=resolved_parameter_ids, + ) + + +def create_resolved_policy( + session: Session, + policy_input: ResolvedPolicyCreationInput, + *, + canonicalizer: Callable[ + [ResolvedPolicyCreationInput], CanonicalPolicyContent + ] = canonicalize_policy, +) -> PolicyCreationResult: + content = canonicalizer(policy_input) + created_policy_id = create_policy( + session, + policy_input, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if created_policy_id is not None: + create_parameter_values( + session, + policy_id=created_policy_id, + policy_input=policy_input, + ) + return PolicyCreationResult(policy_id=created_policy_id, created=True) + + existing = read_policy_by_content_identity( + session, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if existing is None: + raise PolicyCreationIntegrityError( + "policy hash conflict did not resolve to a stored policy" + ) + model_version = read_model_version(session, existing.tax_benefit_model_version_id) + if model_version is None: + raise PolicyCreationIntegrityError( + "stored policy references an absent model version" + ) + stored_input = stored_policy_creation_input( + existing, + model_version, + read_parameter_values(session, existing.id), + ) + if canonical_policy_document(stored_input) != content.document: + raise PolicyContentHashCollisionError( + "stored policy content differs for the same canonical version and hash" + ) + return PolicyCreationResult(policy_id=existing.id, created=False) + + +def read_complete_policy( + session: Session, *, country_id: str, policy_id: UUID +) -> PolicyRead: + policy = read_policy_row(session, country_id=country_id, policy_id=policy_id) + if policy is None: + raise PolicyNotFoundError(f"policy {policy_id} was not found") + values = policy_parameter_values_by_policy( + [policy.id], read_parameter_values_with_names(session, [policy.id]) + ) + return policy_read(policy, values) + + +def read_policy_page( + session: Session, + *, + country_id: str, + tax_benefit_model_id: UUID | None, + offset: int, + limit: int, +) -> PolicyPage: + rows = read_policy_rows( + session, + country_id=country_id, + tax_benefit_model_id=tax_benefit_model_id, + offset=offset, + limit=limit, + ) + displayed_rows = rows[:limit] + policy_ids = [policy.id for policy in displayed_rows] + values = policy_parameter_values_by_policy( + policy_ids, read_parameter_values_with_names(session, policy_ids) + ) + return policy_page(rows, values, offset=offset, limit=limit) + + +def mirror_legacy_policy_in_session( + session: Session, + snapshot: LegacyPolicySnapshot, + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + country_package_versions: Mapping[str, str] = COUNTRY_PACKAGE_VERSIONS, +) -> LegacyPolicyPersistenceResult: + existing = read_legacy_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + lock=True, + ) + if existing is not None: + verify_legacy_policy_mapping( + existing, source_policy_hash=snapshot.source_policy_hash + ) + + selected = read_policy_catalog( + session, + snapshot.country_id, + running_policyengine_version=running_policyengine_version, + ) + policy_json = snapshot.policy_json + assert isinstance(policy_json, dict) + parameters = read_parameters_by_name( + session, + model_version_id=selected.model_version.id, + names=set(policy_json), + ) + policy_input = translate_legacy_policy( + snapshot, + selected=selected, + parameters=parameters, + country_package_versions=country_package_versions, + ) + policy_result = create_resolved_policy(session, policy_input) + if existing is not None: + verify_legacy_policy_mapping( + existing, + source_policy_hash=snapshot.source_policy_hash, + expected_policy_id=policy_result.policy_id, + ) + return LegacyPolicyPersistenceResult( + policy_id=existing.policy_id, + policy_created=False, + mapping_created=False, + ) + + mapping_id = create_legacy_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + source_policy_hash=snapshot.source_policy_hash, + policy_id=policy_result.policy_id, + ) + if mapping_id is not None: + return LegacyPolicyPersistenceResult( + policy_id=policy_result.policy_id, + policy_created=policy_result.created, + mapping_created=True, + ) + concurrent = read_legacy_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + lock=False, + ) + if concurrent is None: + raise LegacyPolicyMappingIntegrityError( + "legacy policy mapping conflict did not resolve to a stored row" + ) + verify_legacy_policy_mapping( + concurrent, + source_policy_hash=snapshot.source_policy_hash, + expected_policy_id=policy_result.policy_id, + ) + return LegacyPolicyPersistenceResult( + policy_id=concurrent.policy_id, + policy_created=False, + mapping_created=False, + ) + + +class V2PolicyService: + """Sequence policy operations while delegating database lifetime management.""" + + def __init__( + self, + database_session: PolicyDatabaseSession, + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + ) -> None: + self._database_session = database_session + self._running_policyengine_version = running_policyengine_version + + def create_policy( + self, policy_input: NativePolicyCreationInput + ) -> NativePolicyCreation: + content = PolicyCreationInput.model_validate( + policy_input.model_dump(exclude={"policyengine_version"}) + ) + with self._database_session.transaction() as session: + resolved = resolve_policy_creation_input( + session, + content, + policyengine_version=policy_input.policyengine_version, + running_policyengine_version=self._running_policyengine_version, + ) + persisted = create_resolved_policy(session, resolved) + item = read_complete_policy( + session, + country_id=policy_input.country_id, + policy_id=persisted.policy_id, + ) + return NativePolicyCreation(item=item, created=persisted.created) + + def get_policy(self, *, country_id: str, policy_id: UUID) -> PolicyRead: + with self._database_session.read() as session: + return read_complete_policy( + session, country_id=country_id, policy_id=policy_id + ) + + def list_policies( + self, + *, + country_id: str, + tax_benefit_model_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> PolicyPage: + with self._database_session.read() as session: + return read_policy_page( + session, + country_id=country_id, + tax_benefit_model_id=tax_benefit_model_id, + offset=offset, + limit=limit, + ) + + def mirror_legacy_policy( + self, snapshot: LegacyPolicySnapshot + ) -> LegacyPolicyPersistenceResult: + with self._database_session.transaction() as session: + return mirror_legacy_policy_in_session( + session, + snapshot, + running_policyengine_version=self._running_policyengine_version, + ) diff --git a/policyengine_api/services/v2/policies/transformations.py b/policyengine_api/services/v2/policies/transformations.py new file mode 100644 index 000000000..576ae30c8 --- /dev/null +++ b/policyengine_api/services/v2/policies/transformations.py @@ -0,0 +1,287 @@ +"""Pure representation transformations for v2 policy operations.""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import date, datetime, time, timezone +from decimal import Decimal +import hashlib +import json +from typing import Any +from uuid import UUID + +from policyengine_core.periods import period as parse_policyengine_period # type: ignore[import-untyped] + +from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog +from policyengine_api.data.v2.models import ( + Parameter, + ParameterValue, + Policy, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.policies.types import ( + CanonicalPolicyContent, + LegacyPolicySnapshot, + PolicyCreationInput, + PolicyPage, + PolicyParameterValueInput, + PolicyParameterValueRead, + PolicyRead, + ResolvedPolicyCreationInput, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyTranslationError, + validate_policy_catalog, +) + + +POLICY_CANONICALIZATION_VERSION = 1 + + +def _canonical_number(value: int | float) -> str: + number = Decimal(str(value)) + if number.is_zero(): + return "0" + rendered = format(number, "f") + if "." in rendered: + rendered = rendered.rstrip("0").rstrip(".") + return rendered + + +def _canonical_json(value: Any) -> str: + if value is None: + return "null" + if type(value) is bool: + return "true" if value else "false" + if type(value) in {int, float}: + return _canonical_number(value) + if type(value) is str: + return json.dumps(value, ensure_ascii=True, allow_nan=False) + if type(value) is list: + return "[" + ",".join(_canonical_json(item) for item in value) + "]" + if type(value) is dict: + members = ( + f"{json.dumps(key, ensure_ascii=True)}:{_canonical_json(value[key])}" + for key in sorted(value) + ) + return "{" + ",".join(members) + "}" + raise TypeError("canonical policy content contains a non-JSON value") + + +def canonical_utc_datetime(value: datetime) -> str: + utc_value = value.astimezone(timezone.utc) + return utc_value.isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def canonical_policy_document(policy_input: ResolvedPolicyCreationInput) -> bytes: + parameter_values = sorted( + policy_input.parameter_values, + key=lambda value: ( + str(value.parameter_id), + canonical_utc_datetime(value.start_date), + "" if value.end_date is None else canonical_utc_datetime(value.end_date), + ), + ) + document = { + "canonicalization_version": POLICY_CANONICALIZATION_VERSION, + "country_id": policy_input.country_id, + "tax_benefit_model_id": str(policy_input.tax_benefit_model_id), + "tax_benefit_model_version_id": str(policy_input.tax_benefit_model_version_id), + "parameter_values": [ + { + "parameter_id": str(value.parameter_id), + "value": value.value, + "start_date": canonical_utc_datetime(value.start_date), + "end_date": ( + None + if value.end_date is None + else canonical_utc_datetime(value.end_date) + ), + } + for value in parameter_values + ], + } + return _canonical_json(document).encode("ascii") + + +def canonicalize_policy( + policy_input: ResolvedPolicyCreationInput, +) -> CanonicalPolicyContent: + document = canonical_policy_document(policy_input) + return CanonicalPolicyContent( + version=POLICY_CANONICALIZATION_VERSION, + document=document, + content_hash=hashlib.sha256(document).hexdigest(), + ) + + +def stored_policy_creation_input( + policy: Policy, + model_version: TaxBenefitModelVersion, + values: list[ParameterValue], +) -> ResolvedPolicyCreationInput: + return ResolvedPolicyCreationInput.model_validate( + { + "country_id": policy.country_id, + "tax_benefit_model_id": policy.tax_benefit_model_id, + "tax_benefit_model_version_id": policy.tax_benefit_model_version_id, + "policyengine_version": model_version.version, + "parameter_values": [ + { + "parameter_id": value.parameter_id, + "value": value.value_json, + "start_date": value.start_date, + "end_date": value.end_date, + } + for value in values + ], + } + ) + + +def policy_parameter_values_by_policy( + policy_ids: list[UUID], + rows: list[tuple[ParameterValue, str]], +) -> dict[UUID, tuple[PolicyParameterValueRead, ...]]: + grouped: dict[UUID, list[PolicyParameterValueRead]] = { + policy_id: [] for policy_id in policy_ids + } + for value, parameter_name in rows: + if value.policy_id is None: + continue + grouped[value.policy_id].append( + PolicyParameterValueRead( + id=value.id, + parameter_id=value.parameter_id, + parameter_name=parameter_name, + value=value.value_json, + start_date=value.start_date, + end_date=value.end_date, + ) + ) + return {policy_id: tuple(values) for policy_id, values in grouped.items()} + + +def policy_read( + policy: Policy, + values: dict[UUID, tuple[PolicyParameterValueRead, ...]], +) -> PolicyRead: + return PolicyRead( + id=policy.id, + country_id=policy.country_id, + tax_benefit_model_id=policy.tax_benefit_model_id, + tax_benefit_model_version_id=policy.tax_benefit_model_version_id, + created_at=policy.created_at, + updated_at=policy.updated_at, + parameter_values=values.get(policy.id, ()), + ) + + +def policy_page( + rows: list[Policy], + values: dict[UUID, tuple[PolicyParameterValueRead, ...]], + *, + offset: int, + limit: int, +) -> PolicyPage: + policies = rows[:limit] + return PolicyPage( + items=tuple(policy_read(policy, values) for policy in policies), + offset=offset, + limit=limit, + has_more=len(rows) > limit, + ) + + +def _utc_midnight(value: str) -> datetime: + try: + parsed = date.fromisoformat(value) + except ValueError as error: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} is invalid" + ) from error + return datetime.combine(parsed, time.min, tzinfo=timezone.utc) + + +def parse_legacy_period(value: str) -> tuple[datetime, datetime]: + if not value or value != value.strip(): + raise LegacyPolicyTranslationError("legacy period must be non-empty") + if "." in value: + parts = value.split(".") + if len(parts) != 2: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} must contain one date range" + ) + start_date, end_date = (_utc_midnight(item) for item in parts) + else: + try: + parsed = parse_policyengine_period(value) + start_date = _utc_midnight(str(parsed.start)) + end_date = _utc_midnight(str(parsed.stop)) + except (TypeError, ValueError) as error: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} is invalid" + ) from error + if end_date < start_date: + raise LegacyPolicyTranslationError( + f"legacy period {value!r} ends before it starts" + ) + return start_date, end_date + + +def translate_legacy_policy( + snapshot: LegacyPolicySnapshot, + *, + selected: SelectedCatalog, + parameters: Mapping[str, Parameter], + country_package_versions: Mapping[str, str], +) -> ResolvedPolicyCreationInput: + expected_api_version = country_package_versions.get(snapshot.country_id) + if expected_api_version is None or snapshot.api_version != expected_api_version: + raise LegacyPolicyTranslationError( + "legacy policy api_version does not match the running country package" + ) + policy_json = snapshot.policy_json + assert isinstance(policy_json, dict) + parameter_names = set(policy_json) + if set(parameters) != parameter_names: + raise LegacyPolicyTranslationError( + "every legacy parameter path must exist in the running catalog" + ) + + parameter_values: list[PolicyParameterValueInput] = [] + for parameter_name in sorted(parameter_names): + period_values = policy_json[parameter_name] + if type(period_values) is not dict: + raise LegacyPolicyTranslationError( + f"legacy parameter {parameter_name!r} must contain period/value entries" + ) + for period_name, value in sorted(period_values.items()): + if type(period_name) is not str: + raise LegacyPolicyTranslationError( + f"legacy parameter {parameter_name!r} has a non-string period" + ) + start_date, end_date = parse_legacy_period(period_name) + parameter_values.append( + PolicyParameterValueInput( + parameter_id=parameters[parameter_name].id, + value=value, + start_date=start_date, + end_date=end_date, + ) + ) + try: + policy_input = PolicyCreationInput( + country_id=snapshot.country_id, + tax_benefit_model_id=selected.model.id, + parameter_values=parameter_values, + ) + except ValueError as error: + raise LegacyPolicyTranslationError( + "legacy parameter periods or values conflict" + ) from error + return validate_policy_catalog( + policy_input, + selected=selected, + resolved_parameter_ids={parameter.id for parameter in parameters.values()}, + ) diff --git a/policyengine_api/services/v2/policies/types.py b/policyengine_api/services/v2/policies/types.py new file mode 100644 index 000000000..86dbc19bd --- /dev/null +++ b/policyengine_api/services/v2/policies/types.py @@ -0,0 +1,172 @@ +"""Framework-independent data exchanged by v2 policy layers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import Annotated, Any +from uuid import UUID + +from pydantic import ( + AfterValidator, + BaseModel, + BeforeValidator, + ConfigDict, + Field, + JsonValue, + field_validator, + model_validator, +) + +from policyengine_api.query_parameters import CountryId, PolicyEngineVersion +from policyengine_api.services.v2.policies.validators import ( + MAXIMUM_POLICY_PARAMETER_VALUES, + normalize_utc, + require_json_value, +) + + +StrictJsonValue = Annotated[JsonValue, BeforeValidator(require_json_value)] +UtcDateTime = Annotated[datetime, AfterValidator(normalize_utc)] + + +class StrictPolicyInput(BaseModel): + """Reject undeclared policy input and non-finite numeric coercion.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, frozen=True) + + +class PolicyParameterValueInput(StrictPolicyInput): + """One normalized effective value for a catalog parameter UUID.""" + + parameter_id: UUID + value: StrictJsonValue + start_date: UtcDateTime + end_date: UtcDateTime | None = None + + @model_validator(mode="after") + def validate_period(self) -> "PolicyParameterValueInput": + if self.end_date is not None and self.end_date < self.start_date: + raise ValueError("end_date must not precede start_date") + return self + + +class PolicyCreationInput(StrictPolicyInput): + """Complete immutable policy content accepted from any source.""" + + country_id: CountryId + tax_benefit_model_id: UUID + parameter_values: Annotated[ + list[PolicyParameterValueInput], + Field(max_length=MAXIMUM_POLICY_PARAMETER_VALUES), + ] + + @field_validator("parameter_values") + @classmethod + def reject_duplicate_effective_values( + cls, + values: list[PolicyParameterValueInput], + ) -> list[PolicyParameterValueInput]: + identities: set[tuple[UUID, datetime]] = set() + for value in values: + identity = (value.parameter_id, value.start_date) + if identity in identities: + raise ValueError( + "parameter_values must not repeat a parameter_id/start_date" + ) + identities.add(identity) + return values + + +class NativePolicyCreationInput(PolicyCreationInput): + """Native policy content plus an optional catalog-version selection.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class ResolvedPolicyCreationInput(PolicyCreationInput): + """Validated content bound to one exact initialized catalog version.""" + + policyengine_version: PolicyEngineVersion + tax_benefit_model_version_id: UUID + + +class LegacyPolicySnapshot(StrictPolicyInput): + """Detached committed fields required by the v2 policy mirror.""" + + country_id: CountryId + legacy_policy_id: Annotated[int, Field(ge=0)] + label: Annotated[str, Field(max_length=255)] | None = None + api_version: Annotated[str, Field(min_length=1, max_length=255)] + policy_json: StrictJsonValue + source_policy_hash: Annotated[str, Field(min_length=1, max_length=255)] + + @field_validator("policy_json") + @classmethod + def require_parameter_mapping(cls, value: object) -> object: + if type(value) is not dict: + raise ValueError("legacy policy_json must be a parameter-path object") + return value + + +@dataclass(frozen=True) +class CanonicalPolicyContent: + """Canonical bytes and SHA-256 identity for one resolved policy.""" + + version: int + document: bytes + content_hash: str + + +@dataclass(frozen=True) +class PolicyCreationResult: + """New or deduplicated immutable policy identity.""" + + policy_id: UUID + created: bool + + +@dataclass(frozen=True) +class LegacyPolicyPersistenceResult: + """Destination identity and insertion outcomes for one mirror attempt.""" + + policy_id: UUID + policy_created: bool + mapping_created: bool + + +@dataclass(frozen=True) +class PolicyParameterValueRead: + id: UUID + parameter_id: UUID + parameter_name: str + value: Any + start_date: datetime + end_date: datetime | None + + +@dataclass(frozen=True) +class PolicyRead: + id: UUID + country_id: str + tax_benefit_model_id: UUID + tax_benefit_model_version_id: UUID + created_at: datetime + updated_at: datetime + parameter_values: tuple[PolicyParameterValueRead, ...] + + +@dataclass(frozen=True) +class PolicyPage: + items: tuple[PolicyRead, ...] + offset: int + limit: int + has_more: bool + + +@dataclass(frozen=True) +class NativePolicyCreation: + """Complete policy read plus whether this request inserted it.""" + + item: PolicyRead + created: bool diff --git a/policyengine_api/services/v2/policies/validators.py b/policyengine_api/services/v2/policies/validators.py new file mode 100644 index 000000000..822db40b5 --- /dev/null +++ b/policyengine_api/services/v2/policies/validators.py @@ -0,0 +1,134 @@ +"""Database-independent validation for v2 policy operations.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import math +from typing import TYPE_CHECKING, Any +from uuid import UUID + +from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog +from policyengine_api.data.v2.models import LegacyPolicyMapping + +if TYPE_CHECKING: + from policyengine_api.services.v2.policies.types import ( + PolicyCreationInput, + ResolvedPolicyCreationInput, + ) + + +MAXIMUM_POLICY_PARAMETER_VALUES = 1_000 +MAXIMUM_JSON_NESTING = 100 + + +class PolicyCatalogValidationError(ValueError): + """Raised when policy content does not belong to the selected catalog.""" + + +class PolicyNotFoundError(LookupError): + """Raised when a policy UUID is absent from the selected country.""" + + +class PolicyCreationIntegrityError(RuntimeError): + """Raised when stored policy content cannot support safe deduplication.""" + + +class PolicyContentHashCollisionError(PolicyCreationIntegrityError): + """Raised when equal version/hash keys identify different canonical bytes.""" + + +class LegacyPolicyMappingIntegrityError(RuntimeError): + """Raised when one immutable v1 identity maps inconsistently.""" + + +class LegacyPolicyTranslationError(ValueError): + """Raised when committed v1 content cannot be interpreted exactly.""" + + +def require_json_value( + value: Any, + *, + depth: int = 0, + containers: frozenset[int] = frozenset(), +) -> Any: + """Reject values that cannot be represented as standards-compliant JSON.""" + + if depth > MAXIMUM_JSON_NESTING: + raise ValueError("JSON values must not exceed 100 nested containers") + if value is None or type(value) in {str, bool, int}: + return value + if type(value) is float: + if not math.isfinite(value): + raise ValueError("JSON numbers must be finite") + return value + if type(value) not in {list, dict}: + raise ValueError("value must contain only standards-compliant JSON types") + identity = id(value) + if identity in containers: + raise ValueError("JSON values must not contain reference cycles") + nested_containers = containers | {identity} + if type(value) is list: + for item in value: + require_json_value(item, depth=depth + 1, containers=nested_containers) + return value + for key, item in value.items(): + if type(key) is not str: + raise ValueError("JSON object keys must be strings") + require_json_value(item, depth=depth + 1, containers=nested_containers) + return value + + +def normalize_utc(value: datetime) -> datetime: + """Require a time-zone-aware value and normalize it to UTC.""" + + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("effective dates must include a UTC offset") + return value.astimezone(timezone.utc) + + +def validate_policy_catalog( + policy_input: "PolicyCreationInput", + *, + selected: SelectedCatalog, + resolved_parameter_ids: set[UUID], +) -> "ResolvedPolicyCreationInput": + """Validate preloaded catalog records and bind them to policy content.""" + + from policyengine_api.services.v2.policies.types import ResolvedPolicyCreationInput + + if policy_input.tax_benefit_model_id != selected.model.id: + raise PolicyCatalogValidationError( + "tax_benefit_model_id does not match the selected country catalog" + ) + requested_parameter_ids = { + value.parameter_id for value in policy_input.parameter_values + } + if resolved_parameter_ids != requested_parameter_ids: + raise PolicyCatalogValidationError( + "every parameter_id must belong to the selected model version" + ) + return ResolvedPolicyCreationInput( + country_id=policy_input.country_id, + tax_benefit_model_id=selected.model.id, + tax_benefit_model_version_id=selected.model_version.id, + policyengine_version=selected.policyengine_version, + parameter_values=policy_input.parameter_values, + ) + + +def verify_legacy_policy_mapping( + mapping: LegacyPolicyMapping, + *, + source_policy_hash: str, + expected_policy_id: UUID | None = None, +) -> None: + """Validate an existing mapping without performing SQL.""" + + if mapping.source_policy_hash != source_policy_hash: + raise LegacyPolicyMappingIntegrityError( + "legacy policy identity was presented with a different source hash" + ) + if expected_policy_id is not None and mapping.policy_id != expected_policy_id: + raise LegacyPolicyMappingIntegrityError( + "legacy policy mapping does not match translated immutable content" + ) diff --git a/policyengine_api/services/v2/user_policies/__init__.py b/policyengine_api/services/v2/user_policies/__init__.py new file mode 100644 index 000000000..77569938d --- /dev/null +++ b/policyengine_api/services/v2/user_policies/__init__.py @@ -0,0 +1 @@ +"""Application services for mutable v2 user-policy associations.""" diff --git a/policyengine_api/services/v2/user_policies/database_connectors/__init__.py b/policyengine_api/services/v2/user_policies/database_connectors/__init__.py new file mode 100644 index 000000000..47b6f091b --- /dev/null +++ b/policyengine_api/services/v2/user_policies/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for v2 user-policy services.""" diff --git a/policyengine_api/services/v2/user_policies/database_connectors/creates.py b/policyengine_api/services/v2/user_policies/database_connectors/creates.py new file mode 100644 index 000000000..e8d48701f --- /dev/null +++ b/policyengine_api/services/v2/user_policies/database_connectors/creates.py @@ -0,0 +1,71 @@ +"""Database inserts used by v2 user-policy operations.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, col + +from policyengine_api.data.v2.models import ( + LegacyUserMapping, + LegacyUserPolicyMapping, + User, + UserPolicy, +) +from policyengine_api.services.v2.user_policies.types import UserPolicyCreationInput + + +def create_user_policy( + session: Session, association_input: UserPolicyCreationInput +) -> UserPolicy: + association = UserPolicy(**association_input.model_dump()) + session.add(association) + session.flush() + session.refresh(association) + return association + + +def create_transition_user(session: Session, *, primary_country: str) -> User: + user = User(primary_country=primary_country) + session.add(user) + session.flush() + return user + + +def create_legacy_user_mapping( + session: Session, *, legacy_user_id: str, user_id: UUID +) -> UUID | None: + return session.execute( + insert(LegacyUserMapping) + .values(legacy_user_id=legacy_user_id, user_id=user_id) + .on_conflict_do_nothing(index_elements=[LegacyUserMapping.legacy_user_id]) + .returning(col(LegacyUserMapping.user_id)) + ).scalar_one_or_none() + + +def create_legacy_user_policy_mapping( + session: Session, + *, + country_id: str, + legacy_user_policy_id: int, + user_policy_id: UUID, + source_revision: int, + fingerprint_version: int, + fingerprint: str, +) -> UUID | None: + return session.execute( + insert(LegacyUserPolicyMapping) + .values( + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + user_policy_id=user_policy_id, + last_applied_source_revision=source_revision, + fingerprint_version=fingerprint_version, + fingerprint_sha256=fingerprint, + ) + .on_conflict_do_nothing( + constraint="uq_legacy_user_policy_mappings_country_legacy" + ) + .returning(col(LegacyUserPolicyMapping.id)) + ).scalar_one_or_none() diff --git a/policyengine_api/services/v2/user_policies/database_connectors/deletes.py b/policyengine_api/services/v2/user_policies/database_connectors/deletes.py new file mode 100644 index 000000000..7b0a077de --- /dev/null +++ b/policyengine_api/services/v2/user_policies/database_connectors/deletes.py @@ -0,0 +1,17 @@ +"""Database deletes used by v2 user-policy operations.""" + +from __future__ import annotations + +from sqlmodel import Session + +from policyengine_api.data.v2.models import User, UserPolicy + + +def delete_user_policy(session: Session, association: UserPolicy) -> None: + session.delete(association) + session.flush() + + +def delete_transition_user(session: Session, user: User) -> None: + session.delete(user) + session.flush() diff --git a/policyengine_api/services/v2/user_policies/database_connectors/reads.py b/policyengine_api/services/v2/user_policies/database_connectors/reads.py new file mode 100644 index 000000000..66a12624f --- /dev/null +++ b/policyengine_api/services/v2/user_policies/database_connectors/reads.py @@ -0,0 +1,96 @@ +"""Database selections used by v2 user-policy operations.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import ( + LegacyUserMapping, + LegacyUserPolicyMapping, + Policy, + User, + UserPolicy, +) + + +def read_user(session: Session, user_id: UUID) -> User | None: + return session.get(User, user_id) + + +def read_policy_for_association(session: Session, policy_id: UUID) -> Policy | None: + return session.exec(select(Policy).where(Policy.id == policy_id)).one_or_none() + + +def read_legacy_user_mapping( + session: Session, legacy_user_id: str, *, lock: bool +) -> LegacyUserMapping | None: + statement = select(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == legacy_user_id + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def read_legacy_user_policy_mapping( + session: Session, + *, + country_id: str, + legacy_user_policy_id: int, + lock: bool, +) -> LegacyUserPolicyMapping | None: + statement = select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.country_id == country_id, + LegacyUserPolicyMapping.legacy_user_policy_id == legacy_user_policy_id, + ) + if lock: + statement = statement.with_for_update() + return session.exec(statement).one_or_none() + + +def read_mapped_user_policy( + session: Session, mapping: LegacyUserPolicyMapping +) -> UserPolicy | None: + return session.exec( + select(UserPolicy).where( + UserPolicy.id == mapping.user_policy_id, + UserPolicy.country_id == mapping.country_id, + ) + ).one_or_none() + + +def read_user_policy_row( + session: Session, *, country_id: str, association_id: UUID +) -> UserPolicy | None: + return session.exec( + select(UserPolicy).where( + UserPolicy.id == association_id, + UserPolicy.country_id == country_id, + ) + ).one_or_none() + + +def read_user_policy_rows( + session: Session, + *, + country_id: str, + user_id: UUID, + policy_id: UUID | None, + offset: int, + limit: int, +) -> list[UserPolicy]: + statement = select(UserPolicy).where( + UserPolicy.country_id == country_id, + UserPolicy.user_id == user_id, + ) + if policy_id is not None: + statement = statement.where(UserPolicy.policy_id == policy_id) + return list( + session.exec( + statement.order_by(col(UserPolicy.created_at), col(UserPolicy.id)) + .offset(offset) + .limit(limit + 1) + ).all() + ) diff --git a/policyengine_api/services/v2/user_policies/database_connectors/updates.py b/policyengine_api/services/v2/user_policies/database_connectors/updates.py new file mode 100644 index 000000000..c58fd4d12 --- /dev/null +++ b/policyengine_api/services/v2/user_policies/database_connectors/updates.py @@ -0,0 +1,44 @@ +"""Database updates used by v2 user-policy operations.""" + +from __future__ import annotations + +from sqlmodel import Session + +from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy +from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.services.v2.user_policies.types import UserPolicyUpdateInput + + +def update_user_policy( + session: Session, + association: UserPolicy, + association_input: UserPolicyUpdateInput, +) -> UserPolicy: + for field_name, value in association_input.model_dump(exclude_unset=True).items(): + setattr(association, field_name, value) + association.updated_at = utc_now() + session.add(association) + session.flush() + session.refresh(association) + return association + + +def update_legacy_user_policy_state( + session: Session, + *, + association: UserPolicy, + mapping: LegacyUserPolicyMapping, + reform_label: str | None, + update_name: bool, + fingerprint: str, + source_revision: int, +) -> None: + if update_name: + association.name = reform_label + association.updated_at = utc_now() + session.add(association) + mapping.fingerprint_sha256 = fingerprint + mapping.last_applied_source_revision = source_revision + mapping.updated_at = utc_now() + session.add(mapping) + session.flush() diff --git a/policyengine_api/services/v2/user_policies/database_session.py b/policyengine_api/services/v2/user_policies/database_session.py new file mode 100644 index 000000000..f7576c23b --- /dev/null +++ b/policyengine_api/services/v2/user_policies/database_session.py @@ -0,0 +1,26 @@ +"""Database session lifetime management for v2 user-policy services.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + + +class UserPolicyDatabaseSession: + """Open read sessions and transactions for user-policy services.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + with self._session_factory() as session: + yield session + + @contextmanager + def transaction(self) -> Iterator[Session]: + with self._session_factory.begin() as session: + yield session diff --git a/policyengine_api/services/v2/user_policies/services.py b/policyengine_api/services/v2/user_policies/services.py new file mode 100644 index 000000000..ac71912d4 --- /dev/null +++ b/policyengine_api/services/v2/user_policies/services.py @@ -0,0 +1,394 @@ +"""Router-facing and legacy-mirroring services for v2 user-policy resources.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlmodel import Session + +from policyengine_api.data.v2.models import LegacyUserPolicyMapping +from policyengine_api.services.v2.policies.services import ( + mirror_legacy_policy_in_session, +) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.user_policies.database_connectors.creates import ( + create_legacy_user_mapping, + create_legacy_user_policy_mapping, + create_transition_user, + create_user_policy, +) +from policyengine_api.services.v2.user_policies.database_connectors.deletes import ( + delete_transition_user, + delete_user_policy, +) +from policyengine_api.services.v2.user_policies.database_connectors.reads import ( + read_legacy_user_mapping, + read_legacy_user_policy_mapping, + read_mapped_user_policy, + read_policy_for_association, + read_user, + read_user_policy_row, + read_user_policy_rows, +) +from policyengine_api.services.v2.user_policies.database_connectors.updates import ( + update_legacy_user_policy_state, + update_user_policy, +) +from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, +) +from policyengine_api.services.v2.user_policies.transformations import ( + USER_POLICY_FINGERPRINT_VERSION, + fingerprint_legacy_user_policy, + legacy_name_requires_update, + project_legacy_user_policy, + user_policy_page, + user_policy_read, +) +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyMappingAction, + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, + UserPolicyCreationInput, + UserPolicyPage, + UserPolicyRead, + UserPolicyUpdateInput, +) +from policyengine_api.services.v2.user_policies.validators import ( + LegacyUserPolicyIntegrityError, + require_user_policy, + validate_association_creation, + validate_existing_legacy_user_mapping, + validate_existing_legacy_user_policy_mapping, + validate_legacy_user_mapping_conflict, + validate_legacy_user_policy_input, +) + + +def read_complete_user_policy( + session: Session, *, country_id: str, association_id: UUID +) -> UserPolicyRead: + association = require_user_policy( + read_user_policy_row( + session, + country_id=country_id, + association_id=association_id, + ), + association_id=association_id, + ) + return user_policy_read(association) + + +def read_user_policy_page( + session: Session, + *, + country_id: str, + user_id: UUID, + policy_id: UUID | None, + offset: int, + limit: int, +) -> UserPolicyPage: + rows = read_user_policy_rows( + session, + country_id=country_id, + user_id=user_id, + policy_id=policy_id, + offset=offset, + limit=limit, + ) + return user_policy_page(rows, offset=offset, limit=limit) + + +def resolve_legacy_user_id( + session: Session, + *, + legacy_user_id: str, + primary_country: str, +) -> UUID: + existing = read_legacy_user_mapping(session, legacy_user_id, lock=True) + if existing is not None: + return validate_existing_legacy_user_mapping( + existing, + user=read_user(session, existing.user_id), + ) + + user = create_transition_user(session, primary_country=primary_country) + created_user_id = create_legacy_user_mapping( + session, + legacy_user_id=legacy_user_id, + user_id=user.id, + ) + if created_user_id is not None: + return created_user_id + + delete_transition_user(session, user) + concurrent = read_legacy_user_mapping(session, legacy_user_id, lock=False) + return validate_legacy_user_mapping_conflict( + concurrent, + user=( + read_user(session, concurrent.user_id) if concurrent is not None else None + ), + ) + + +def apply_existing_legacy_user_policy_mapping( + session: Session, + *, + mapping: LegacyUserPolicyMapping, + country_id: str, + reform_label: str | None, + fingerprint: str, + user_id: UUID, + policy_id: UUID, + changed_fields: frozenset[str], + source_revision: int, +) -> LegacyUserPolicyPersistenceResult: + action, association = validate_existing_legacy_user_policy_mapping( + mapping, + association=read_mapped_user_policy(session, mapping), + country_id=country_id, + fingerprint=fingerprint, + user_id=user_id, + policy_id=policy_id, + source_revision=source_revision, + fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, + ) + if action in { + LegacyUserPolicyMappingAction.STALE, + LegacyUserPolicyMappingAction.REPLAY, + }: + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=False, + mapping_created=False, + ) + + association_updated = legacy_name_requires_update( + association, + reform_label=reform_label, + changed_fields=changed_fields, + ) + update_legacy_user_policy_state( + session, + association=association, + mapping=mapping, + reform_label=reform_label, + update_name=association_updated, + fingerprint=fingerprint, + source_revision=source_revision, + ) + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=policy_id, + association_created=False, + association_updated=association_updated, + mapping_created=False, + ) + + +def persist_legacy_user_policy_mapping( + session: Session, + *, + country_id: str, + legacy_user_policy_id: int, + reform_label: str | None, + projection: UserPolicyCreationInput, + fingerprint: str, + source_revision: int, + changed_fields: frozenset[str], +) -> LegacyUserPolicyPersistenceResult: + existing = read_legacy_user_policy_mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + lock=True, + ) + if existing is not None: + return apply_existing_legacy_user_policy_mapping( + session, + mapping=existing, + country_id=country_id, + reform_label=reform_label, + fingerprint=fingerprint, + user_id=projection.user_id, + policy_id=projection.policy_id, + changed_fields=changed_fields, + source_revision=source_revision, + ) + + association = create_user_policy(session, projection) + mapping_id = create_legacy_user_policy_mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + user_policy_id=association.id, + source_revision=source_revision, + fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, + fingerprint=fingerprint, + ) + if mapping_id is not None: + return LegacyUserPolicyPersistenceResult( + association_id=association.id, + policy_id=projection.policy_id, + association_created=True, + association_updated=False, + mapping_created=True, + ) + + delete_user_policy(session, association) + concurrent = read_legacy_user_policy_mapping( + session, + country_id=country_id, + legacy_user_policy_id=legacy_user_policy_id, + lock=False, + ) + if concurrent is None: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping conflict did not resolve to a stored row" + ) + return apply_existing_legacy_user_policy_mapping( + session, + mapping=concurrent, + country_id=country_id, + reform_label=reform_label, + fingerprint=fingerprint, + user_id=projection.user_id, + policy_id=projection.policy_id, + changed_fields=changed_fields, + source_revision=source_revision, + ) + + +def mirror_legacy_user_policy_in_session( + session: Session, + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str] = frozenset(), +) -> LegacyUserPolicyPersistenceResult: + validate_legacy_user_policy_input( + snapshot, + reform_snapshot, + source_revision=source_revision, + ) + policy_result = mirror_legacy_policy_in_session(session, reform_snapshot) + user_id = resolve_legacy_user_id( + session, + legacy_user_id=snapshot.user_id, + primary_country=snapshot.country_id, + ) + projection = project_legacy_user_policy( + snapshot, + user_id=user_id, + policy_id=policy_result.policy_id, + ) + return persist_legacy_user_policy_mapping( + session, + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + reform_label=snapshot.reform_label, + projection=projection, + fingerprint=fingerprint_legacy_user_policy(snapshot), + source_revision=source_revision, + changed_fields=changed_fields, + ) + + +class V2UserPolicyService: + """Sequence association operations through explicit database sessions.""" + + def __init__(self, database_session: UserPolicyDatabaseSession) -> None: + self._database_session = database_session + + def create_user_policy( + self, association_input: UserPolicyCreationInput + ) -> UserPolicyRead: + with self._database_session.transaction() as session: + policy = read_policy_for_association(session, association_input.policy_id) + validate_association_creation( + association_input, + user=read_user(session, association_input.user_id), + policy=policy, + ) + return user_policy_read(create_user_policy(session, association_input)) + + def get_user_policy( + self, *, country_id: str, association_id: UUID + ) -> UserPolicyRead: + with self._database_session.read() as session: + return read_complete_user_policy( + session, + country_id=country_id, + association_id=association_id, + ) + + def list_user_policies( + self, + *, + country_id: str, + user_id: UUID, + policy_id: UUID | None = None, + offset: int = 0, + limit: int = 100, + ) -> UserPolicyPage: + with self._database_session.read() as session: + return read_user_policy_page( + session, + country_id=country_id, + user_id=user_id, + policy_id=policy_id, + offset=offset, + limit=limit, + ) + + def patch_user_policy( + self, + *, + country_id: str, + association_id: UUID, + association_input: UserPolicyUpdateInput, + ) -> UserPolicyRead: + with self._database_session.transaction() as session: + association = require_user_policy( + read_user_policy_row( + session, + country_id=country_id, + association_id=association_id, + ), + association_id=association_id, + ) + return user_policy_read( + update_user_policy(session, association, association_input) + ) + + def delete_user_policy(self, *, country_id: str, association_id: UUID) -> None: + with self._database_session.transaction() as session: + association = require_user_policy( + read_user_policy_row( + session, + country_id=country_id, + association_id=association_id, + ), + association_id=association_id, + ) + delete_user_policy(session, association) + + def mirror_legacy_user_policy( + self, + snapshot: LegacyUserPolicySnapshot, + reform_snapshot: LegacyPolicySnapshot, + *, + source_revision: int, + changed_fields: frozenset[str], + ) -> LegacyUserPolicyPersistenceResult: + with self._database_session.transaction() as session: + return mirror_legacy_user_policy_in_session( + session, + snapshot, + reform_snapshot, + source_revision=source_revision, + changed_fields=changed_fields, + ) diff --git a/policyengine_api/services/v2/user_policies/transformations.py b/policyengine_api/services/v2/user_policies/transformations.py new file mode 100644 index 000000000..615059f07 --- /dev/null +++ b/policyengine_api/services/v2/user_policies/transformations.py @@ -0,0 +1,85 @@ +"""Pure representation transformations for v2 user-policy operations.""" + +from __future__ import annotations + +from hashlib import sha256 +import json +from uuid import UUID + +from policyengine_api.data.v2.models import UserPolicy +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicySnapshot, + UserPolicyCreationInput, + UserPolicyPage, + UserPolicyRead, +) + + +USER_POLICY_FINGERPRINT_VERSION = 1 + + +def user_policy_read(association: UserPolicy) -> UserPolicyRead: + return UserPolicyRead( + id=association.id, + country_id=association.country_id, + user_id=association.user_id, + policy_id=association.policy_id, + name=association.name, + description=association.description, + created_at=association.created_at, + updated_at=association.updated_at, + ) + + +def user_policy_page( + rows: list[UserPolicy], *, offset: int, limit: int +) -> UserPolicyPage: + return UserPolicyPage( + items=tuple(user_policy_read(row) for row in rows[:limit]), + offset=offset, + limit=limit, + has_more=len(rows) > limit, + ) + + +def fingerprint_legacy_user_policy(snapshot: LegacyUserPolicySnapshot) -> str: + """Hash every committed source field through deterministic JSON.""" + + document = { + "fingerprint_version": USER_POLICY_FINGERPRINT_VERSION, + **snapshot.model_dump(mode="json"), + } + encoded = json.dumps( + document, + ensure_ascii=True, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def project_legacy_user_policy( + snapshot: LegacyUserPolicySnapshot, + *, + user_id: UUID, + policy_id: UUID, +) -> UserPolicyCreationInput: + """Map v1 presentation data onto an association, never core policy content.""" + + return UserPolicyCreationInput( + country_id=snapshot.country_id, + user_id=user_id, + policy_id=policy_id, + name=snapshot.reform_label, + description=None, + ) + + +def legacy_name_requires_update( + association: UserPolicy, + *, + reform_label: str | None, + changed_fields: frozenset[str], +) -> bool: + return "reform_label" in changed_fields and association.name != reform_label diff --git a/policyengine_api/services/v2/user_policies/types.py b/policyengine_api/services/v2/user_policies/types.py new file mode 100644 index 000000000..67f70cc6a --- /dev/null +++ b/policyengine_api/services/v2/user_policies/types.py @@ -0,0 +1,99 @@ +"""Framework-independent data exchanged by v2 user-policy layers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from enum import StrEnum +from typing import Annotated +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator + +from policyengine_api.query_parameters import ( + CountryId, + LegacyUserId, + ResourceId, + UserId, +) + + +class StrictAssociationInput(BaseModel): + """Reject fields outside the reviewed association contract.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class UserPolicyCreationInput(StrictAssociationInput): + country_id: CountryId + user_id: UserId + policy_id: ResourceId + name: Annotated[str, StringConstraints(max_length=255)] | None = None + description: str | None = None + + +class UserPolicyUpdateInput(StrictAssociationInput): + name: Annotated[str, StringConstraints(max_length=255)] | None = None + description: str | None = None + + @model_validator(mode="after") + def require_supplied_field(self) -> "UserPolicyUpdateInput": + if not self.model_fields_set.intersection({"name", "description"}): + raise ValueError("At least one of name or description must be supplied") + return self + + +class LegacyUserPolicySnapshot(StrictAssociationInput): + """Detached complete committed v1 saved-policy row.""" + + country_id: CountryId + legacy_user_policy_id: Annotated[int, Field(ge=0)] + reform_id: Annotated[int, Field(ge=0)] + reform_label: Annotated[str, Field(max_length=255)] | None = None + baseline_id: Annotated[int, Field(ge=0)] + baseline_label: Annotated[str, Field(max_length=255)] | None = None + user_id: LegacyUserId + year: Annotated[str, Field(max_length=32)] + geography: Annotated[str, Field(max_length=255)] + dataset: Annotated[str, Field(max_length=255)] | None = None + number_of_provisions: Annotated[int, Field(ge=0)] + api_version: Annotated[str, Field(max_length=32)] + added_date: int + updated_date: int + budgetary_impact: Annotated[str, Field(max_length=255)] | None = None + type: Annotated[str, Field(max_length=255)] | None = None + + +class LegacyUserPolicyMappingAction(StrEnum): + STALE = "stale" + REPLAY = "replay" + UPDATE = "update" + + +@dataclass(frozen=True) +class UserPolicyRead: + id: UUID + country_id: str + user_id: UUID + policy_id: UUID + name: str | None + description: str | None + created_at: datetime + updated_at: datetime + + +@dataclass(frozen=True) +class UserPolicyPage: + items: tuple[UserPolicyRead, ...] + offset: int + limit: int + has_more: bool + + +@dataclass(frozen=True) +class LegacyUserPolicyPersistenceResult: + association_id: UUID + policy_id: UUID + association_created: bool + association_updated: bool + mapping_created: bool diff --git a/policyengine_api/services/v2/user_policies/validators.py b/policyengine_api/services/v2/user_policies/validators.py new file mode 100644 index 000000000..1d895ecba --- /dev/null +++ b/policyengine_api/services/v2/user_policies/validators.py @@ -0,0 +1,155 @@ +"""Database-independent validation for v2 user-policy operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from uuid import UUID + +from policyengine_api.data.v2.models import ( + LegacyUserMapping, + LegacyUserPolicyMapping, + Policy, + User, + UserPolicy, +) +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyMappingAction, +) + +if TYPE_CHECKING: + from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot + from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicySnapshot, + UserPolicyCreationInput, + ) + + +class UserPolicyNotFoundError(LookupError): + """Raised when an association is absent from the selected country.""" + + +class AssociationPolicyNotFoundError(LookupError): + """Raised when an association references an unknown policy UUID.""" + + +class AssociationUserNotFoundError(LookupError): + """Raised when an association references an unknown v2 user UUID.""" + + +class AssociationCountryConflictError(ValueError): + """Raised when an association and its referenced policy differ by country.""" + + +class LegacyUserPolicyIntegrityError(RuntimeError): + """Raised when source, policy, association, or mapping identity conflicts.""" + + +def require_user_policy( + association: UserPolicy | None, *, association_id: UUID +) -> UserPolicy: + if association is None: + raise UserPolicyNotFoundError( + f"user-policy association {association_id} was not found" + ) + return association + + +def validate_association_creation( + association_input: "UserPolicyCreationInput", + *, + user: User | None, + policy: Policy | None, +) -> None: + if user is None: + raise AssociationUserNotFoundError( + f"user {association_input.user_id} was not found" + ) + if policy is None: + raise AssociationPolicyNotFoundError( + f"policy {association_input.policy_id} was not found" + ) + if policy.country_id != association_input.country_id: + raise AssociationCountryConflictError( + "Association country_id must match the referenced policy" + ) + + +def validate_existing_legacy_user_mapping( + mapping: LegacyUserMapping, *, user: User | None +) -> UUID: + if user is None: + raise LegacyUserPolicyIntegrityError( + "legacy user mapping has no referenced v2 user" + ) + return mapping.user_id + + +def validate_legacy_user_mapping_conflict( + mapping: LegacyUserMapping | None, *, user: User | None +) -> UUID: + if mapping is None or user is None: + raise LegacyUserPolicyIntegrityError( + "legacy user mapping conflict did not resolve to a v2 user" + ) + return mapping.user_id + + +def validate_legacy_user_policy_input( + snapshot: "LegacyUserPolicySnapshot", + reform_snapshot: "LegacyPolicySnapshot", + *, + source_revision: int, +) -> None: + if source_revision <= 0: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy source revision must be positive" + ) + if ( + snapshot.country_id != reform_snapshot.country_id + or snapshot.reform_id != reform_snapshot.legacy_policy_id + ): + raise LegacyUserPolicyIntegrityError( + "saved policy does not reference the supplied reform snapshot" + ) + + +def validate_existing_legacy_user_policy_mapping( + mapping: LegacyUserPolicyMapping, + *, + association: UserPolicy | None, + country_id: str, + fingerprint: str, + user_id: UUID, + policy_id: UUID, + source_revision: int, + fingerprint_version: int, +) -> tuple[LegacyUserPolicyMappingAction, UserPolicy]: + if association is None: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping has no association" + ) + if ( + association.policy_id != policy_id + or association.country_id != country_id + or association.user_id != user_id + ): + raise LegacyUserPolicyIntegrityError( + "legacy user-policy mapping conflicts with immutable association fields" + ) + if mapping.fingerprint_version != fingerprint_version: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy fingerprint version is unsupported" + ) + if source_revision < mapping.last_applied_source_revision: + return LegacyUserPolicyMappingAction.STALE, association + if source_revision == mapping.last_applied_source_revision: + if mapping.fingerprint_sha256 != fingerprint: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy revision conflicts with its stored fingerprint" + ) + return LegacyUserPolicyMappingAction.REPLAY, association + if source_revision != mapping.last_applied_source_revision + 1: + raise LegacyUserPolicyIntegrityError( + "legacy user-policy revision has an unapplied predecessor" + ) + return LegacyUserPolicyMappingAction.UPDATE, association diff --git a/pyproject.toml b/pyproject.toml index 5efe10516..f5e873fbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ dependencies = [ dev = [ "build", "coverage", + "mypy>=1.15,<2", "pytest", "pytest-snapshot", "pytest-timeout", @@ -73,6 +74,22 @@ packages = ["policyengine_api"] [tool.ruff.lint.per-file-ignores] "migrations/*/versions/*.py" = ["F401"] +[tool.mypy] +python_version = "3.11" +files = [ + "policyengine_api/query_parameters.py", + "policyengine_api/fastapi_routes/query_parameters.py", + "policyengine_api/fastapi_routes/dependencies.py", + "policyengine_api/fastapi_routes/v2", + "policyengine_api/services/v2", +] +explicit_package_bases = true +follow_imports = "silent" +disallow_untyped_defs = true +warn_return_any = true +warn_unused_ignores = true +show_error_codes = true + [tool.towncrier] package = "policyengine_api" directory = "changelog.d" diff --git a/scripts/guards/migration_contracts.py b/scripts/guards/migration_contracts.py index 699991520..04d3c0918 100644 --- a/scripts/guards/migration_contracts.py +++ b/scripts/guards/migration_contracts.py @@ -83,11 +83,14 @@ def _check_workflows(payload: dict[str, Any]) -> list[str]: violations.append( f"{context}: unknown route_group {request['route_group']!r}" ) - if not request["stable_response_fields"]: + if ( + not request["stable_response_fields"] + and request["expected_status"] != 204 + ): violations.append(f"{context}: stable_response_fields is required") if not request["path"].startswith("/"): violations.append(f"{context}: path must start with /") - if request["expected_status"] not in {200, 201, 202}: + if request["expected_status"] not in {200, 201, 202, 204}: violations.append( f"{context}: unexpected status {request['expected_status']}" ) diff --git a/scripts/qualify_v2_policy_migration.py b/scripts/qualify_v2_policy_migration.py new file mode 100644 index 000000000..841ee20ea --- /dev/null +++ b/scripts/qualify_v2_policy_migration.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Verify predecessor policy data before the Phase 10 revision is applied.""" + +from policyengine_api.data.v2.policy_migration_qualification import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/contract/registry.py b/tests/contract/registry.py index c24e9eb41..0c5c26df8 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -49,6 +49,177 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="policy_resources_v2", + current_contract="typed_v2_resources", + future_owner_pr="PR 10: Policy Migration", + requests=( + ContractRequest( + method="POST", + path="/v2/policies?country_id=us", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/v2/policies/{policy_id}?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/v2/policies?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.items", + "result.offset", + "result.limit", + "result.has_more", + ), + route_group="policy", + ), + ), + ), + WorkflowContract( + name="saved_policy_v1_compatibility", + current_contract="api_v1_compatible", + future_owner_pr="PR 10: Policy Migration", + requests=( + ContractRequest( + method="POST", + path="/us/user-policy", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.id", + "result.reform_id", + "result.reform_label", + "result.baseline_id", + "result.user_id", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/us/user-policy/{user_id}", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result", + ), + route_group="policy", + ), + ContractRequest( + method="PUT", + path="/us/user-policy", + expected_status=200, + stable_response_fields=("status", "message", "result.id"), + route_group="policy", + ), + ), + ), + WorkflowContract( + name="user_policy_associations_v2", + current_contract="typed_v2_resources", + future_owner_pr="PR 10: Policy Migration", + requests=( + ContractRequest( + method="POST", + path="/v2/user-policies?country_id=us", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.user_id", + "result.item.policy_id", + "result.item.name", + "result.item.description", + "result.item.created_at", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path=("/v2/user-policies/{association_id}?country_id=us"), + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.user_id", + "result.item.policy_id", + "result.item.name", + "result.item.description", + "result.item.created_at", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/v2/user-policies?country_id=us&user_id={user_id}", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.items", + "result.offset", + "result.limit", + "result.has_more", + ), + route_group="policy", + ), + ContractRequest( + method="PATCH", + path=("/v2/user-policies/{association_id}?country_id=us"), + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.name", + "result.item.description", + "result.item.updated_at", + ), + route_group="policy", + ), + ContractRequest( + method="DELETE", + path=("/v2/user-policies/{association_id}?country_id=us"), + expected_status=204, + stable_response_fields=(), + route_group="policy", + ), + ), + ), WorkflowContract( name="household_save_edit_read", current_contract="api_v1_compatible", diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py index 8a1991d84..4545af6bf 100644 --- a/tests/contract/test_app_v2_workflow_contracts.py +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -9,6 +9,9 @@ def test_app_v2_workflow_contract_registry_is_complete(): assert {workflow.name for workflow in APP_V2_WORKFLOW_CONTRACTS} == { "policy_save_search", + "policy_resources_v2", + "saved_policy_v1_compatibility", + "user_policy_associations_v2", "household_save_edit_read", "household_calculate", "region_selection", @@ -21,7 +24,12 @@ def test_app_v2_workflow_contract_registry_is_complete(): for workflow in APP_V2_WORKFLOW_CONTRACTS: expected_contract = ( "typed_v2_resources" - if workflow.name == "metadata_resources_v2_preview" + if workflow.name + in { + "metadata_resources_v2_preview", + "policy_resources_v2", + "user_policy_associations_v2", + } else "api_v1_compatible" ) assert workflow.current_contract == expected_contract @@ -29,10 +37,10 @@ def test_app_v2_workflow_contract_registry_is_complete(): assert workflow.requests for request in APP_V2_ROUTE_CONTRACTS: - assert request.method in {"GET", "POST", "PUT", "PATCH"} + assert request.method in {"GET", "POST", "PUT", "PATCH", "DELETE"} assert request.path.startswith("/") - assert request.expected_status in {200, 201, 202} - assert request.stable_response_fields + assert request.expected_status in {200, 201, 202, 204} + assert request.stable_response_fields or request.expected_status == 204 assert request.route_group in ROUTE_GROUP_CONFIG_BY_NAME assert all( @@ -44,6 +52,6 @@ def test_app_v2_workflow_contract_registry_is_complete(): } == { request.path for workflow in APP_V2_WORKFLOW_CONTRACTS - if workflow.name == "metadata_resources_v2_preview" + if workflow.current_contract == "typed_v2_resources" for request in workflow.requests } diff --git a/tests/contract/test_policy_v2_compatibility.py b/tests/contract/test_policy_v2_compatibility.py new file mode 100644 index 000000000..b8bfaec68 --- /dev/null +++ b/tests/contract/test_policy_v2_compatibility.py @@ -0,0 +1,66 @@ +"""Compatibility boundaries between app-v2 labels and native v2 resources.""" + +from __future__ import annotations + +from uuid import UUID + +import pytest + +from policyengine_api.data.v2.models import Policy +from policyengine_api.fastapi_routes.v2.policies.request_models import ( + PolicyCreateRequest, +) +from policyengine_api.services.v2.user_policies.transformations import ( + project_legacy_user_policy, +) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot + + +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +USER_ID = UUID("00000000-0000-0000-0000-000000000070") + + +def test_app_v2_reform_label_maps_to_association_name_only() -> None: + snapshot = LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=10, + reform_id=2, + reform_label="User-visible app label", + baseline_id=1, + baseline_label="Current law", + user_id="auth0|one", + year="2026", + geography="us", + dataset=None, + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + + projection = project_legacy_user_policy( + snapshot, + user_id=USER_ID, + policy_id=POLICY_ID, + ) + + assert projection.name == "User-visible app label" + assert projection.description is None + assert projection.user_id == USER_ID + assert projection.policy_id == POLICY_ID + assert "name" not in Policy.__table__.c + assert "description" not in Policy.__table__.c + + +def test_core_policy_request_rejects_association_presentation_fields() -> None: + with pytest.raises(ValueError): + PolicyCreateRequest.model_validate( + { + "country_id": "us", + "tax_benefit_model_id": str(POLICY_ID), + "parameter_values": [], + "name": "User-visible app label", + } + ) diff --git a/tests/contract/test_v1_route_contracts.py b/tests/contract/test_v1_route_contracts.py index bc06f381f..2296369a4 100644 --- a/tests/contract/test_v1_route_contracts.py +++ b/tests/contract/test_v1_route_contracts.py @@ -14,7 +14,9 @@ ReportOutput, ReportOutputRun, Simulation, + UserPolicy, ) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot from policyengine_api.extensions import cache from policyengine_api.routes.household_routes import household_bp from policyengine_api.routes.policy_routes import policy_bp @@ -28,6 +30,10 @@ HouseholdCalculationResult, ) from policyengine_api.services.simulation_service import SimulationCreateResult +from policyengine_api.services.user_policy_service import ( + UserPolicyCreateResult, + UserPolicyUpdateResult, +) from tests.contract.clients import ( ASGIContractClient, ContractClient, @@ -184,12 +190,30 @@ def _resolved_path(path: str) -> str: .replace("{household_id}", "456") .replace("{simulation_id}", "11") .replace("{report_id}", "33") + .replace("{user_id}", "auth0|one") ) def _json_payload(contract: ContractRequest) -> dict | None: if contract.path == "/us/policy": return {"label": "Utah reform", "data": {"gov.example.parameter": 1}} + if contract.path == "/us/user-policy" and contract.method == "POST": + return { + "reform_id": 22, + "reform_label": "Tax reform", + "baseline_id": 2, + "baseline_label": "Current law", + "user_id": "auth0|one", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1", + "added_date": 1, + "updated_date": 2, + } + if contract.path == "/us/user-policy" and contract.method == "PUT": + return {"id": 10, "reform_label": "Updated tax reform"} if contract.path == "/us/household": return {"label": "Empty household", "data": {}} if contract.path == "/us/household/{household_id}": @@ -219,6 +243,42 @@ def _fake_country(): def _patched_route_dependencies(): stack = ExitStack() + saved_policy = UserPolicy( + id=10, + country_id="us", + reform_id=22, + reform_label="Tax reform", + baseline_id=2, + baseline_label="Current law", + user_id="auth0|one", + year="2026", + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + saved_snapshot = LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=10, + reform_id=22, + reform_label="Tax reform", + baseline_id=2, + baseline_label="Current law", + user_id="auth0|one", + year="2026", + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) stack.enter_context( patch( "policyengine_api.routes.policy_routes.policy_service.get_policy", @@ -232,6 +292,32 @@ def _patched_route_dependencies(): ), ) ) + stack.enter_context( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.create_or_get_user_policy", + return_value=UserPolicyCreateResult( + user_policy=saved_policy, + created=True, + snapshot=saved_snapshot, + ), + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.list_user_policies", + return_value=[saved_policy], + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.update_user_policy", + return_value=UserPolicyUpdateResult( + user_policy=saved_policy, + snapshot=saved_snapshot, + changed_fields=frozenset({"reform_label"}), + ), + ) + ) stack.enter_context( patch( "policyengine_api.routes.policy_routes.policy_service.set_policy", @@ -397,6 +483,30 @@ def _expected_subset(contract: ContractRequest) -> dict: "message": "Policies found", "result": [{"id": 123, "label": "Tax reform"}], } + if contract.path == "/us/user-policy" and contract.method == "POST": + return { + "status": "ok", + "message": "Record created successfully", + "result": { + "id": 10, + "reform_id": 22, + "reform_label": "Tax reform", + "baseline_id": 2, + "user_id": "auth0|one", + }, + } + if contract.path == "/us/user-policy/{user_id}": + return { + "status": "ok", + "message": None, + "result": [{"id": 10, "reform_label": "Tax reform"}], + } + if contract.path == "/us/user-policy" and contract.method == "PUT": + return { + "status": "ok", + "message": "Record updated successfully", + "result": {"id": 10}, + } if contract.path == "/us/household": return {"status": "ok", "message": None, "result": {"household_id": 456}} if contract.path == "/us/household/{household_id}" and contract.method == "PUT": diff --git a/tests/integration/test_alembic_mysql_lifecycle.py b/tests/integration/test_alembic_mysql_lifecycle.py index 9e4f0344f..4428d45f3 100644 --- a/tests/integration/test_alembic_mysql_lifecycle.py +++ b/tests/integration/test_alembic_mysql_lifecycle.py @@ -30,7 +30,7 @@ BASELINE_REVISION = "eafc2a547a4e" -PREVIOUS_REVISION = "17bb32415f97" +PREVIOUS_REVISION = "1914c0422236" def _deployed_question_table() -> Table: @@ -97,6 +97,10 @@ def test_fresh_upgrade_check_downgrade_and_reupgrade(): } assert reform_impact_columns["dataset"]["default"] is None assert reform_impact_columns["execution_id"]["nullable"] is False + assert "mirror_revision" in { + column["name"] for column in inspector.get_columns("user_policies") + } + assert "user_policy_mirror_events" in inspector.get_table_names() command.downgrade(config, BASELINE_REVISION) assert "question" in inspect(engine).get_table_names() @@ -142,6 +146,10 @@ def test_pending_upgrade_commits_head_revision(monkeypatch): } assert reform_impact_columns["dataset"]["default"] is None assert reform_impact_columns["execution_id"]["nullable"] is False + assert "mirror_revision" in { + column["name"] for column in inspector.get_columns("user_policies") + } + assert "user_policy_mirror_events" in inspector.get_table_names() finally: command.upgrade(config, "head") engine.dispose() diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index dd24c82fd..6978d5187 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -23,7 +23,8 @@ BASELINE_REVISION = "f5ef4347cb2a" -HEAD_REVISION = "68b4a5ae5dc5" +PREVIOUS_HEAD_REVISION = "c21c4a807a49" +HEAD_REVISION = "af34023a728f" V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) @@ -76,6 +77,21 @@ def _assert_head(engine) -> None: ] assert canonical_index["unique"] assert canonical_index["column_names"] == ["parameter_id", "start_date"] + policy_value_constraint = next( + constraint + for constraint in inspect(engine).get_unique_constraints("parameter_values") + if constraint["name"] == "uq_parameter_values_policy_parameter_start_date" + ) + assert policy_value_constraint["column_names"] == [ + "policy_id", + "parameter_id", + "start_date", + ] + assert { + "legacy_policy_mappings", + "legacy_user_mappings", + "legacy_user_policy_mappings", + } <= set(inspect(engine).get_table_names(schema="public")) def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: @@ -93,6 +109,35 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: command.check(config) _assert_head(engine) + command.downgrade(config, PREVIOUS_HEAD_REVISION) + with engine.connect() as connection: + context = MigrationContext.configure(connection) + assert context.get_current_revision() == PREVIOUS_HEAD_REVISION + assert "legacy_user_mappings" not in inspect(engine).get_table_names( + schema="public" + ) + user_policy_id = next( + column + for column in inspect(engine).get_columns( + "user_policies", + schema="public", + ) + if column["name"] == "user_id" + ) + assert isinstance(user_policy_id["type"], sa.String) + user_columns = { + column["name"]: column + for column in inspect(engine).get_columns("users", schema="public") + } + assert all( + not user_columns[field_name]["nullable"] + for field_name in ("first_name", "last_name", "email") + ) + + command.upgrade(config, "head") + command.check(config) + _assert_head(engine) + command.downgrade(config, BASELINE_REVISION) with engine.connect() as connection: context = MigrationContext.configure(connection) diff --git a/tests/integration/test_live_phase10_staging.py b/tests/integration/test_live_phase10_staging.py new file mode 100644 index 000000000..630a1cc7d --- /dev/null +++ b/tests/integration/test_live_phase10_staging.py @@ -0,0 +1,498 @@ +"""Stateful activation and rollback checks for the isolated staging databases.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import json +import os +from pathlib import Path +from uuid import uuid4 + +import httpx +import pytest +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL +from sqlalchemy.pool import NullPool + + +V1_ONLY_COUNTRIES = ("ca", "ng", "il") +SYNTHETIC_PARAMETER = "gov.states.ut.tax.income.rate" + +pytestmark = pytest.mark.skipif( + os.environ.get("RUN_PHASE10_STAGING_EXERCISE") != "1", + reason="live Phase 10 staging exercise was not explicitly selected", +) + + +def _required_environment(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required for the live Phase 10 staging exercise") + return value + + +def _state_path() -> Path: + return Path(_required_environment("PHASE10_STATE_PATH")) + + +def _v1_engine(): + password = _required_environment("POLICYENGINE_DB_READONLY_PASSWORD") + return create_engine( + URL.create( + "mysql+pymysql", + username="policyengine_schema_reader", + password=password, + host="127.0.0.1", + port=3307, + database="policyengine", + ), + poolclass=NullPool, + ) + + +def _v2_engine(): + return create_engine( + _required_environment("V2_MIGRATION_DATABASE_URL"), + poolclass=NullPool, + ) + + +def _client(setting: str) -> httpx.Client: + return httpx.Client( + base_url=_required_environment(setting).rstrip("/"), + timeout=90, + follow_redirects=True, + ) + + +def _counts(v1_engine, v2_engine) -> dict[str, int]: + with v1_engine.connect() as connection: + v1_counts = { + "v1_policies": connection.scalar(text("SELECT COUNT(*) FROM policy")), + "v1_user_policies": connection.scalar( + text("SELECT COUNT(*) FROM user_policies") + ), + "v1_pending_mirror_events": connection.scalar( + text( + "SELECT COUNT(*) FROM user_policy_mirror_events " + "WHERE processed_at IS NULL" + ) + ), + } + with v2_engine.connect() as connection: + v2_counts = { + "v2_policies": connection.scalar(text("SELECT COUNT(*) FROM policies")), + "v2_user_policies": connection.scalar( + text("SELECT COUNT(*) FROM user_policies") + ), + "v2_policy_mappings": connection.scalar( + text("SELECT COUNT(*) FROM legacy_policy_mappings") + ), + "v2_user_policy_mappings": connection.scalar( + text("SELECT COUNT(*) FROM legacy_user_policy_mappings") + ), + } + return {key: int(value) for key, value in {**v1_counts, **v2_counts}.items()} + + +def _policy_payload(label: str, value: float) -> dict[str, object]: + return { + "label": label, + "data": { + SYNTHETIC_PARAMETER: { + "2026-01-01.2100-12-31": value, + } + }, + } + + +def _saved_policy_payload( + *, probe_id: str, reform_id: int, reform_label: str +) -> dict[str, object]: + return { + "reform_id": reform_id, + "reform_label": reform_label, + "baseline_id": reform_id, + "baseline_label": "Synthetic staging baseline", + "user_id": f"phase10-staging-{probe_id}", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 1, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": "phase10-staging", + } + + +def _assert_v1_policy_identity(response: httpx.Response, expected_id: int) -> None: + assert response.status_code == 200, response.text[:500] + item = response.json()["result"] + assert item["id"] == expected_id + assert isinstance(item["id"], int) + assert "uuid" not in item + assert "v2_policy_id" not in item + + +def test_live_phase10_activation_failure_and_retry(integration_probe_id: str) -> None: + """Activate immediate mirroring and prove deterministic retry semantics.""" + + probe_id = integration_probe_id.replace("/", "-") + unique_value = 0.04 + (int(uuid4().hex[:6], 16) % 5000) / 1_000_000 + v1_engine = _v1_engine() + v2_engine = _v2_engine() + evidence: dict[str, object] = { + "environment": "staging", + "started_at": datetime.now(timezone.utc).isoformat(), + "probe_id": probe_id, + "revisions": { + "cloud_sql_only": _required_environment("PHASE10_CLOUD_SQL_REVISION"), + "dual_write": _required_environment("PHASE10_DUAL_WRITE_REVISION"), + "controlled_failure": _required_environment("PHASE10_FAILURE_REVISION"), + }, + "activation_selectors": { + "ROUTE_IMPL_POLICY": "fastapi_native", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "dual_write", + }, + "status_summary": {}, + } + try: + evidence["counts_before"] = _counts(v1_engine, v2_engine) + first_label = f"Phase 10 failure retry {probe_id}" + first_payload = _policy_payload(first_label, unique_value) + + with _client("PHASE10_FAILURE_API_BASE_URL") as failure_client: + failed = failure_client.post("/us/policy", json=first_payload) + assert failed.status_code == 503, failed.text[:500] + evidence["status_summary"]["controlled_supabase_failure"] = 503 + + with _client("API_BASE_URL") as active_client: + retried = active_client.post("/us/policy", json=first_payload) + assert retried.status_code == 200, retried.text[:500] + first_legacy_id = retried.json()["result"]["policy_id"] + assert isinstance(first_legacy_id, int) + evidence["status_summary"]["policy_retry"] = 200 + + _assert_v1_policy_identity( + active_client.get(f"/us/policy/{first_legacy_id}"), + first_legacy_id, + ) + + with v2_engine.connect() as connection: + first_v2_id = connection.scalar( + text( + "SELECT policy_id FROM legacy_policy_mappings " + "WHERE country_id = 'us' AND legacy_policy_id = :legacy_id" + ), + {"legacy_id": first_legacy_id}, + ) + assert first_v2_id is not None + + equivalent_payload = _policy_payload( + f"Phase 10 equivalent label {probe_id}", + unique_value, + ) + equivalent = active_client.post("/us/policy", json=equivalent_payload) + assert equivalent.status_code == 201, equivalent.text[:500] + equivalent_legacy_id = equivalent.json()["result"]["policy_id"] + with v2_engine.connect() as connection: + equivalent_v2_id = connection.scalar( + text( + "SELECT policy_id FROM legacy_policy_mappings " + "WHERE country_id = 'us' AND legacy_policy_id = :legacy_id" + ), + {"legacy_id": equivalent_legacy_id}, + ) + assert equivalent_v2_id == first_v2_id + evidence["status_summary"]["equivalent_content_deduplication"] = 201 + + saved_label = f"Phase 10 saved failure {probe_id}" + saved_payload = _saved_policy_payload( + probe_id=probe_id, + reform_id=first_legacy_id, + reform_label=saved_label, + ) + + with _client("PHASE10_FAILURE_API_BASE_URL") as failure_client: + failed_saved = failure_client.post("/us/user-policy", json=saved_payload) + assert failed_saved.status_code == 503, failed_saved.text[:500] + evidence["status_summary"]["saved_policy_controlled_failure"] = 503 + + with _client("API_BASE_URL") as active_client: + retried_saved = active_client.post("/us/user-policy", json=saved_payload) + assert retried_saved.status_code == 200, retried_saved.text[:500] + saved_policy_id = retried_saved.json()["result"]["id"] + + with v1_engine.connect() as connection: + events = connection.execute( + text( + "SELECT source_revision, processed_at " + "FROM user_policy_mirror_events " + "WHERE country_id = 'us' " + "AND legacy_user_policy_id = :legacy_id " + "ORDER BY source_revision" + ), + {"legacy_id": saved_policy_id}, + ).all() + assert [event.source_revision for event in events] == [1, 2] + assert all(event.processed_at is not None for event in events) + assert events[0].processed_at <= events[1].processed_at + + with v2_engine.connect() as connection: + saved_mapping = connection.execute( + text( + "SELECT m.user_policy_id, m.last_applied_source_revision, " + "m.fingerprint_sha256, a.user_id, a.name, a.description " + "FROM legacy_user_policy_mappings m " + "JOIN user_policies a ON a.id = m.user_policy_id " + "WHERE m.country_id = 'us' " + "AND m.legacy_user_policy_id = :legacy_id" + ), + {"legacy_id": saved_policy_id}, + ).one() + mapped_legacy_user_id = connection.scalar( + text( + "SELECT legacy_user_id FROM legacy_user_mappings " + "WHERE user_id = :user_id" + ), + {"user_id": saved_mapping.user_id}, + ) + assert saved_mapping.last_applied_source_revision == 2 + assert mapped_legacy_user_id == saved_payload["user_id"] + assert saved_mapping.name == saved_label + assert saved_mapping.description is None + + renamed_label = f"Phase 10 saved renamed {probe_id}" + renamed = active_client.put( + "/us/user-policy", + json={"id": saved_policy_id, "reform_label": renamed_label}, + ) + assert renamed.status_code == 200, renamed.text[:500] + with v2_engine.connect() as connection: + renamed_mapping = connection.execute( + text( + "SELECT m.last_applied_source_revision, " + "m.fingerprint_sha256, a.name, a.description " + "FROM legacy_user_policy_mappings m " + "JOIN user_policies a ON a.id = m.user_policy_id " + "WHERE m.country_id = 'us' " + "AND m.legacy_user_policy_id = :legacy_id" + ), + {"legacy_id": saved_policy_id}, + ).one() + assert renamed_mapping.last_applied_source_revision == 3 + assert renamed_mapping.name == renamed_label + assert renamed_mapping.description is None + + v1_only_update = active_client.put( + "/us/user-policy", + json={"id": saved_policy_id, "number_of_provisions": 2}, + ) + assert v1_only_update.status_code == 200, v1_only_update.text[:500] + with v2_engine.connect() as connection: + v1_only_mapping = connection.execute( + text( + "SELECT m.last_applied_source_revision, " + "m.fingerprint_sha256, a.name, a.description " + "FROM legacy_user_policy_mappings m " + "JOIN user_policies a ON a.id = m.user_policy_id " + "WHERE m.country_id = 'us' " + "AND m.legacy_user_policy_id = :legacy_id" + ), + {"legacy_id": saved_policy_id}, + ).one() + assert v1_only_mapping.last_applied_source_revision == 4 + assert ( + v1_only_mapping.fingerprint_sha256 != renamed_mapping.fingerprint_sha256 + ) + assert v1_only_mapping.name == renamed_label + assert v1_only_mapping.description is None + + uk_parameters = active_client.get( + "/v2/parameters", + params={"country_id": "uk", "limit": 1}, + ) + assert uk_parameters.status_code == 200, uk_parameters.text[:500] + uk_parameter_name = uk_parameters.json()["result"]["items"][0]["name"] + uk_policy = active_client.post( + "/uk/policy", + json={ + "label": f"Phase 10 UK {probe_id}", + "data": { + uk_parameter_name: { + "2026-01-01.2100-12-31": unique_value, + } + }, + }, + ) + assert uk_policy.status_code == 201, uk_policy.text[:500] + uk_legacy_id = uk_policy.json()["result"]["policy_id"] + with v2_engine.connect() as connection: + uk_mapping_count = connection.scalar( + text( + "SELECT COUNT(*) FROM legacy_policy_mappings " + "WHERE country_id = 'uk' AND legacy_policy_id = :legacy_id" + ), + {"legacy_id": uk_legacy_id}, + ) + assert uk_mapping_count == 1 + + with _client("PHASE10_FAILURE_API_BASE_URL") as failure_client: + for country_id in V1_ONLY_COUNTRIES: + unsupported_policy = failure_client.post( + f"/{country_id}/policy", + json=_policy_payload( + f"Phase 10 {country_id} v1-only {probe_id}", + unique_value, + ), + ) + assert unsupported_policy.status_code == 201, unsupported_policy.text[ + :500 + ] + unsupported_id = unsupported_policy.json()["result"]["policy_id"] + _assert_v1_policy_identity( + failure_client.get(f"/{country_id}/policy/{unsupported_id}"), + unsupported_id, + ) + unsupported_saved = failure_client.post( + f"/{country_id}/user-policy", + json={ + **_saved_policy_payload( + probe_id=f"{country_id}-{probe_id}", + reform_id=unsupported_id, + reform_label=f"Phase 10 {country_id} saved {probe_id}", + ), + "geography": country_id, + }, + ) + assert unsupported_saved.status_code == 201, unsupported_saved.text[ + :500 + ] + unsupported_saved_id = unsupported_saved.json()["result"]["id"] + with v2_engine.connect() as connection: + mapping_count = connection.scalar( + text( + "SELECT COUNT(*) FROM legacy_policy_mappings " + "WHERE legacy_policy_id = :legacy_id" + ), + {"legacy_id": unsupported_id}, + ) + assert mapping_count == 0 + with v1_engine.connect() as connection: + event_count = connection.scalar( + text( + "SELECT COUNT(*) FROM user_policy_mirror_events " + "WHERE country_id = :country_id " + "AND legacy_user_policy_id = :legacy_id" + ), + { + "country_id": country_id, + "legacy_id": unsupported_saved_id, + }, + ) + assert event_count == 0 + + with v1_engine.connect() as connection: + final_events = connection.execute( + text( + "SELECT source_revision, processed_at " + "FROM user_policy_mirror_events " + "WHERE country_id = 'us' " + "AND legacy_user_policy_id = :legacy_id " + "ORDER BY source_revision" + ), + {"legacy_id": saved_policy_id}, + ).all() + assert [event.source_revision for event in final_events] == [1, 2, 3, 4] + assert all(event.processed_at is not None for event in final_events) + + evidence["status_summary"].update( + { + "saved_policy_retry": 200, + "saved_policy_label_update": 200, + "saved_policy_v1_only_update": 200, + "uk_policy_mirror": 201, + "v1_only_country_writes": 201, + } + ) + evidence["synthetic_ids"] = { + "legacy_policy_id": first_legacy_id, + "equivalent_legacy_policy_id": equivalent_legacy_id, + "v2_policy_id": str(first_v2_id), + "legacy_user_policy_id": saved_policy_id, + "v2_user_policy_id": str(saved_mapping.user_policy_id), + } + evidence["counts_after_activation"] = _counts(v1_engine, v2_engine) + _state_path().write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + finally: + v1_engine.dispose() + v2_engine.dispose() + + +def test_live_phase10_cloud_sql_only_rollback(integration_probe_id: str) -> None: + """Verify rollback removes the v2 dependency without removing v2 state.""" + + evidence = json.loads(_state_path().read_text(encoding="utf-8")) + v1_engine = _v1_engine() + v2_engine = _v2_engine() + try: + unique_value = 0.05 + (int(uuid4().hex[:6], 16) % 5000) / 1_000_000 + rollback_label = f"Phase 10 rollback {integration_probe_id}" + with _client("API_BASE_URL") as rollback_client: + rollback_created = rollback_client.post( + "/us/policy", + json=_policy_payload(rollback_label, unique_value), + ) + assert rollback_created.status_code == 201, rollback_created.text[:500] + rollback_legacy_id = rollback_created.json()["result"]["policy_id"] + _assert_v1_policy_identity( + rollback_client.get(f"/us/policy/{rollback_legacy_id}"), + rollback_legacy_id, + ) + + with v2_engine.connect() as connection: + rollback_mapping_count = connection.scalar( + text( + "SELECT COUNT(*) FROM legacy_policy_mappings " + "WHERE country_id = 'us' AND legacy_policy_id = :legacy_id" + ), + {"legacy_id": rollback_legacy_id}, + ) + assert rollback_mapping_count == 0 + + retained_v2_policy_id = evidence["synthetic_ids"]["v2_policy_id"] + retained = rollback_client.get( + f"/v2/policies/{retained_v2_policy_id}", + params={"country_id": "us"}, + ) + assert retained.status_code == 200, retained.text[:500] + assert retained.json()["result"]["item"]["id"] == retained_v2_policy_id + + evidence["rollback"] = { + "completed_at": datetime.now(timezone.utc).isoformat(), + "revision": _required_environment("PHASE10_CLOUD_SQL_REVISION"), + "selectors": { + "ROUTE_IMPL_POLICY": "fastapi_native", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", + }, + "status_summary": { + "v1_cloud_sql_only_policy_create": 201, + "retained_v2_policy_read": 200, + }, + "synthetic_legacy_policy_id": rollback_legacy_id, + } + evidence["counts_after_rollback"] = _counts(v1_engine, v2_engine) + _state_path().write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + finally: + v1_engine.dispose() + v2_engine.dispose() diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 59ab37914..466745e17 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -48,7 +48,7 @@ def test_live_v2_openapi_describes_every_preview_resource(api_client) -> None: expected_paths.update({"/v2/parameters/children", "/v2/economy-options"}) assert expected_paths <= document["paths"].keys() assert all("get" in document["paths"][path] for path in expected_paths) - assert "MetadataErrorResponse" in document["components"]["schemas"] + assert "V2ErrorResponse" in document["components"]["schemas"] @pytest.mark.parametrize("country_id", ["us", "uk"]) diff --git a/tests/integration/test_live_v2_policies.py b/tests/integration/test_live_v2_policies.py new file mode 100644 index 000000000..e1584c17a --- /dev/null +++ b/tests/integration/test_live_v2_policies.py @@ -0,0 +1,188 @@ +"""Write/read lifecycle checks for deployed native v2 policy resources.""" + +from __future__ import annotations + +import os +from uuid import uuid4 + +from sqlalchemy import create_engine, text +from sqlalchemy.pool import NullPool + + +def _v2_database_url() -> str: + value = os.environ.get("V2_MIGRATION_DATABASE_URL") + if not value: + raise RuntimeError( + "V2_MIGRATION_DATABASE_URL is required for the live v2 probe" + ) + return value + + +def _one_catalog_item(api_client, resource: str, *, country_id: str) -> dict: + response = api_client.get( + f"/v2/{resource}", + params={"country_id": country_id, "limit": 1}, + ) + assert response.status_code == 200, response.text[:500] + items = response.json()["result"]["items"] + assert len(items) == 1 + return items[0] + + +def test_live_native_policy_and_user_policy_lifecycle( + api_client, + integration_probe_id: str, +) -> None: + """Exercise every Phase 10 native route against the staging database.""" + + country_id = "us" + model = _one_catalog_item( + api_client, + "tax-benefit-models", + country_id=country_id, + ) + parameter = _one_catalog_item( + api_client, + "parameters", + country_id=country_id, + ) + user_id = uuid4() + engine = create_engine(_v2_database_url(), poolclass=NullPool) + try: + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO users (id, primary_country) VALUES (:id, :country_id)" + ), + {"id": user_id, "country_id": country_id}, + ) + existing_policy_count = connection.scalar( + text( + "SELECT COUNT(*) FROM policies " + "WHERE country_id = :country_id " + "AND tax_benefit_model_id = :model_id" + ), + {"country_id": country_id, "model_id": model["id"]}, + ) + + policy_body = { + "country_id": country_id, + "tax_benefit_model_id": model["id"], + "parameter_values": [ + { + "parameter_id": parameter["id"], + "value": f"phase10-native-{integration_probe_id}", + "start_date": "2026-01-01T00:00:00Z", + } + ], + } + created = api_client.post( + "/v2/policies", + params={"country_id": country_id}, + json=policy_body, + ) + assert created.status_code == 201, created.text[:500] + created_item = created.json()["result"]["item"] + policy_id = created_item["id"] + assert created_item["country_id"] == country_id + assert created_item["tax_benefit_model_id"] == model["id"] + assert created_item["parameter_values"][0]["parameter_id"] == parameter["id"] + assert ( + created_item["parameter_values"][0]["parameter_name"] == parameter["name"] + ) + assert created_item["created_at"] + assert created_item["updated_at"] + + repeated = api_client.post( + "/v2/policies", + params={"country_id": country_id}, + json=policy_body, + ) + assert repeated.status_code == 200, repeated.text[:500] + assert repeated.json()["result"]["item"]["id"] == policy_id + + detail = api_client.get( + f"/v2/policies/{policy_id}", + params={"country_id": country_id}, + ) + assert detail.status_code == 200, detail.text[:500] + assert detail.json()["result"]["item"] == created_item + + collection = api_client.get( + "/v2/policies", + params={ + "country_id": country_id, + "tax_benefit_model_id": model["id"], + "offset": existing_policy_count, + "limit": 1, + }, + ) + assert collection.status_code == 200, collection.text[:500] + assert policy_id in { + item["id"] for item in collection.json()["result"]["items"] + } + + association_body = { + "country_id": country_id, + "user_id": str(user_id), + "policy_id": policy_id, + "name": f"Phase 10 native {integration_probe_id}", + "description": "Synthetic staging lifecycle record", + } + association = api_client.post( + "/v2/user-policies", + params={"country_id": country_id}, + json=association_body, + ) + assert association.status_code == 201, association.text[:500] + association_item = association.json()["result"]["item"] + association_id = association_item["id"] + + association_detail = api_client.get( + f"/v2/user-policies/{association_id}", + params={"country_id": country_id}, + ) + assert association_detail.status_code == 200, association_detail.text[:500] + assert association_detail.json()["result"]["item"] == association_item + + association_collection = api_client.get( + "/v2/user-policies", + params={ + "country_id": country_id, + "user_id": str(user_id), + "policy_id": policy_id, + }, + ) + assert association_collection.status_code == 200 + assert [ + item["id"] for item in association_collection.json()["result"]["items"] + ] == [association_id] + + updated_name = f"Phase 10 renamed {integration_probe_id}" + patched = api_client.patch( + f"/v2/user-policies/{association_id}", + params={"country_id": country_id}, + json={"name": updated_name}, + ) + assert patched.status_code == 200, patched.text[:500] + patched_item = patched.json()["result"]["item"] + assert patched_item["name"] == updated_name + assert patched_item["description"] == association_body["description"] + + deleted = api_client.delete( + f"/v2/user-policies/{association_id}", + params={"country_id": country_id}, + ) + assert deleted.status_code == 204, deleted.text[:500] + absent = api_client.get( + f"/v2/user-policies/{association_id}", + params={"country_id": country_id}, + ) + assert absent.status_code == 404, absent.text[:500] + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM users WHERE id = :id"), + {"id": user_id}, + ) + engine.dispose() diff --git a/tests/integration/test_v1_policy_dual_write.py b/tests/integration/test_v1_policy_dual_write.py new file mode 100644 index 000000000..69f272605 --- /dev/null +++ b/tests/integration/test_v1_policy_dual_write.py @@ -0,0 +1,265 @@ +"""Cross-database transaction tests for immediate v1 policy mirroring.""" + +from __future__ import annotations + +import os + +import pytest +from sqlalchemy import create_engine, delete, func, select +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v1_models import Policy as V1Policy +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, +) +from policyengine_api.services.v2.policies.services import ( + V2PolicyService, + mirror_legacy_policy_in_session, +) +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.services.policy_mirroring import ( + PolicyMirrorUnavailableError, + mirror_policy_after_commit, +) +from policyengine_api.services.policy_service import PolicyService + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get( + V2_ALEMBIC_DISPOSABLE_TEST, + "", + ), + } + ) + if not settings.disposable_test: + pytest.fail("dual-write tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +def _v1_service(): + engine = create_engine("sqlite://") + with engine.begin() as connection: + connection.exec_driver_sql( + """ + CREATE TABLE policy ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + country_id VARCHAR(3) NOT NULL, + label VARCHAR(255), + api_version VARCHAR(10) NOT NULL, + policy_json JSON NOT NULL, + policy_hash VARCHAR(255) NOT NULL + ) + """ + ) + sessions = sessionmaker(engine, expire_on_commit=False) + return engine, PolicyService(sessions) + + +def _seed_catalog(v2_sessions) -> tuple[object, str]: + with v2_sessions.begin() as session: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version=POLICYENGINE_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.phase10.cross_database_rate", + tax_benefit_model_version=version, + ) + session.add(parameter) + session.flush() + return model.id, parameter.name + + +def _cleanup_v2(v2_engine, model_id) -> None: + if model_id is None: + return + with v2_engine.begin() as connection: + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + connection.execute( + delete(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id.in_(policy_ids) + ) + ) + connection.execute( + delete(ParameterValue).where(ParameterValue.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def _create_v1(service: PolicyService, parameter_name: str): + return service.set_policy( + "us", + "Cross-database policy", + {parameter_name: {"2026": 0.2}}, + prepare_for_mirroring=True, + ) + + +def test_both_commits_and_interrupted_response_retry_resolve_one_mapping() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, v1_service = _v1_service() + model_id = None + try: + model_id, parameter_name = _seed_catalog(v2_sessions) + mirror_service = V2PolicyService(PolicyDatabaseSession(v2_sessions)) + creation = _create_v1(v1_service, parameter_name) + first = mirror_policy_after_commit( + creation.snapshot, + mirror_factory=lambda: mirror_service, + ) + + # Simulate losing the HTTP response after both commits by repeating the + # exact create and mirror operations. + retry_creation = _create_v1(v1_service, parameter_name) + retry = mirror_policy_after_commit( + retry_creation.snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert creation.is_existing_policy is False + assert retry_creation.is_existing_policy is True + assert retry.policy_id == first.policy_id + with v2_sessions() as session: + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 1 + ) + assert session.scalar(select(func.count()).select_from(Policy)) == 1 + assert v1_service.get_policy("us", creation.policy_id) is not None + finally: + _cleanup_v2(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() + + +def test_catalog_failure_leaves_cloud_sql_committed_and_retry_completes() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, v1_service = _v1_service() + model_id = None + parameter_name = "gov.phase10.cross_database_rate" + try: + creation = _create_v1(v1_service, parameter_name) + mirror_service = V2PolicyService(PolicyDatabaseSession(v2_sessions)) + + with pytest.raises(PolicyMirrorUnavailableError): + mirror_policy_after_commit( + creation.snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert v1_service.get_policy("us", creation.policy_id) is not None + with v2_sessions() as session: + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 0 + ) + + model_id, _parameter_name = _seed_catalog(v2_sessions) + retry_creation = _create_v1(v1_service, parameter_name) + result = mirror_policy_after_commit( + retry_creation.snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert retry_creation.is_existing_policy is True + assert result.mapping_created is True + finally: + _cleanup_v2(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() + + +def test_supabase_transaction_failure_rolls_back_and_has_no_background_repair() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, v1_service = _v1_service() + model_id = None + try: + model_id, parameter_name = _seed_catalog(v2_sessions) + creation = _create_v1(v1_service, parameter_name) + + class FailingMirror: + def mirror_legacy_policy(self, snapshot): + with v2_sessions.begin() as session: + mirror_legacy_policy_in_session(session, snapshot) + raise OperationalError( + "forced transaction failure", + {}, + RuntimeError("forced"), + ) + + with pytest.raises(PolicyMirrorUnavailableError): + mirror_policy_after_commit( + creation.snapshot, + mirror_factory=FailingMirror, + ) + + with v2_sessions() as session: + assert session.scalar(select(func.count()).select_from(Policy)) == 0 + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 0 + ) + with v1_engine.connect() as connection: + assert connection.scalar(select(func.count()).select_from(V1Policy)) == 1 + + # No process is scheduled to change this state. Only an explicit retry + # invokes the mirror and creates the missing destination rows. + with v2_sessions() as session: + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 0 + ) + result = mirror_policy_after_commit( + creation.snapshot, + mirror_factory=lambda: V2PolicyService(PolicyDatabaseSession(v2_sessions)), + ) + assert result.mapping_created is True + finally: + _cleanup_v2(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() diff --git a/tests/integration/test_v1_user_policy_dual_write.py b/tests/integration/test_v1_user_policy_dual_write.py new file mode 100644 index 000000000..63383f27b --- /dev/null +++ b/tests/integration/test_v1_user_policy_dual_write.py @@ -0,0 +1,424 @@ +"""Cross-database tests for immediate v1 saved-policy association mirroring.""" + +from __future__ import annotations + +import os + +import pytest +from sqlalchemy import create_engine, delete, func, select +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v1_models import UserPolicyMirrorEvent +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + LegacyUserMapping, + LegacyUserPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + User, + UserPolicy, +) +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, +) +from policyengine_api.services.v2.user_policies.services import V2UserPolicyService +from policyengine_api.services.policy_service import PolicyService +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorUnavailableError, + mirror_pending_user_policy_events_after_commit, + mirror_user_policy_after_commit, +) +from policyengine_api.services.user_policy_service import UserPolicyService + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get( + V2_ALEMBIC_DISPOSABLE_TEST, + "", + ), + } + ) + if not settings.disposable_test: + pytest.fail("saved-policy cross-database tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +def _v1_services(): + engine = create_engine("sqlite://") + with engine.begin() as connection: + connection.exec_driver_sql( + """ + CREATE TABLE policy ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + country_id VARCHAR(3) NOT NULL, + label VARCHAR(255), + api_version VARCHAR(10) NOT NULL, + policy_json JSON NOT NULL, + policy_hash VARCHAR(255) NOT NULL + ) + """ + ) + connection.exec_driver_sql( + """ + CREATE TABLE user_policies ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + country_id VARCHAR(3) NOT NULL, + reform_id INTEGER NOT NULL, + reform_label VARCHAR(255), + baseline_id INTEGER NOT NULL, + baseline_label VARCHAR(255), + user_id VARCHAR(255) NOT NULL, + year VARCHAR(32) NOT NULL, + geography VARCHAR(255) NOT NULL, + dataset VARCHAR(255), + number_of_provisions INTEGER NOT NULL, + api_version VARCHAR(32) NOT NULL, + added_date BIGINT NOT NULL, + updated_date BIGINT NOT NULL, + budgetary_impact VARCHAR(255), + type VARCHAR(255), + mirror_revision BIGINT NOT NULL DEFAULT 0 + ) + """ + ) + UserPolicyMirrorEvent.__table__.create(connection) + sessions = sessionmaker(engine, expire_on_commit=False) + return engine, PolicyService(sessions), UserPolicyService(sessions), sessions + + +def _seed_catalog(sessions): + with sessions.begin() as session: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version=POLICYENGINE_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.phase10.cross_database_saved_rate", + tax_benefit_model_version=version, + ) + session.add(parameter) + session.flush() + return model.id, parameter.name + + +def _saved_values(reform_id: int, **changes): + values = { + "country_id": "us", + "reform_id": reform_id, + "reform_label": "Reform", + "baseline_id": 0, + "baseline_label": "Current law", + "user_id": "auth0|cross-database", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + values.update(changes) + return values + + +def _cleanup(engine, model_id) -> None: + if model_id is None: + return + with engine.begin() as connection: + user_ids = ( + connection.execute( + select(LegacyUserMapping.user_id).where( + LegacyUserMapping.legacy_user_id == "auth0|cross-database" + ) + ) + .scalars() + .all() + ) + connection.execute( + delete(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == "auth0|cross-database" + ) + ) + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + association_ids = select(UserPolicy.id).where( + UserPolicy.policy_id.in_(policy_ids) + ) + connection.execute( + delete(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.user_policy_id.in_(association_ids) + ) + ) + connection.execute( + delete(UserPolicy).where(UserPolicy.policy_id.in_(policy_ids)) + ) + if user_ids: + connection.execute(delete(User).where(User.id.in_(user_ids))) + connection.execute( + delete(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id.in_(policy_ids) + ) + ) + connection.execute( + delete(ParameterValue).where(ParameterValue.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def test_create_update_and_v1_only_change_mirror_one_association() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, policy_service, saved_service, _v1_sessions = _v1_services() + model_id = None + try: + model_id, parameter_name = _seed_catalog(v2_sessions) + reform = policy_service.set_policy( + "us", + "Legacy reform label", + {parameter_name: {"2026": 0.2}}, + prepare_for_mirroring=True, + ) + created = saved_service.create_or_get_user_policy( + _saved_values(reform.policy_id), + record_mirror_event=True, + ) + mirror_service = V2UserPolicyService(UserPolicyDatabaseSession(v2_sessions)) + first = mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + renamed = saved_service.update_user_policy( + "us", + created.user_policy.id, + {"reform_label": "Renamed", "updated_date": 3}, + record_mirror_event=True, + ) + rename_result = mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=renamed.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + v1_only = saved_service.update_user_policy( + "us", + created.user_policy.id, + {"year": "2027", "updated_date": 4}, + record_mirror_event=True, + ) + v1_only_result = mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=v1_only.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert first.association_id == rename_result.association_id + assert first.association_id == v1_only_result.association_id + assert rename_result.association_updated is True + assert v1_only_result.association_updated is False + with v2_sessions() as session: + association = session.get(UserPolicy, first.association_id) + assert association.name == "Renamed" + assert association.description is None + user_mapping = session.scalar(select(LegacyUserMapping)) + assert association.user_id == user_mapping.user_id + assert session.get(User, user_mapping.user_id).primary_country == "us" + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 1 + ) + mapping = session.scalar(select(LegacyUserPolicyMapping)) + assert mapping.last_applied_source_revision == 3 + finally: + _cleanup(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() + + +def test_failure_after_cloud_commit_and_identical_create_retry_are_idempotent() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, policy_service, saved_service, v1_sessions = _v1_services() + model_id = None + parameter_name = "gov.phase10.cross_database_saved_rate" + try: + reform = policy_service.set_policy( + "us", + "Legacy reform label", + {parameter_name: {"2026": 0.2}}, + ) + created = saved_service.create_or_get_user_policy( + _saved_values(reform.policy_id), + record_mirror_event=True, + ) + mirror_service = V2UserPolicyService(UserPolicyDatabaseSession(v2_sessions)) + + with pytest.raises(UserPolicyMirrorUnavailableError): + mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + with v1_sessions() as session: + assert ( + session.scalar( + select(func.count()).select_from(created.user_policy.__class__) + ) + == 1 + ) + event = session.scalar(select(UserPolicyMirrorEvent)) + assert event.processed_at is None + with v2_sessions() as session: + assert session.scalar(select(func.count()).select_from(User)) == 0 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 0 + ) + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 0 + ) + + model_id, _parameter_name = _seed_catalog(v2_sessions) + retry = saved_service.create_or_get_user_policy( + _saved_values(reform.policy_id, number_of_provisions=99), + record_mirror_event=True, + ) + result = mirror_pending_user_policy_events_after_commit( + "us", + retry.user_policy.id, + through_revision=retry.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert retry.created is False + assert retry.user_policy.id == created.user_policy.id + assert result.association_created is False + with v1_sessions() as session: + events = session.scalars(select(UserPolicyMirrorEvent)).all() + assert [event.source_revision for event in events] == [1, 2] + assert all(event.processed_at is not None for event in events) + with v2_sessions() as session: + mapping = session.scalar(select(LegacyUserPolicyMapping)) + assert mapping.last_applied_source_revision == 2 + assert session.scalar(select(func.count()).select_from(User)) == 1 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 1 + ) + finally: + _cleanup(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() + + +def test_destination_commit_replays_when_source_processing_marker_is_missing() -> None: + v2_engine = create_engine(_disposable_url()) + v2_sessions = sessionmaker(v2_engine, class_=Session, expire_on_commit=False) + v1_engine, policy_service, saved_service, v1_sessions = _v1_services() + model_id = None + try: + model_id, parameter_name = _seed_catalog(v2_sessions) + reform = policy_service.set_policy( + "us", + "Legacy reform label", + {parameter_name: {"2026": 0.2}}, + prepare_for_mirroring=True, + ) + created = saved_service.create_or_get_user_policy( + _saved_values(reform.policy_id), + record_mirror_event=True, + ) + mirror_service = V2UserPolicyService(UserPolicyDatabaseSession(v2_sessions)) + + committed = mirror_user_policy_after_commit( + created.snapshot, + reform.snapshot, + source_revision=created.mirror_revision, + mirror_factory=lambda: mirror_service, + ) + with v1_sessions() as session: + event = session.scalar(select(UserPolicyMirrorEvent)) + assert event.processed_at is None + + replayed = mirror_pending_user_policy_events_after_commit( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ) + + assert replayed.association_id == committed.association_id + assert replayed.association_created is False + with v1_sessions() as session: + event = session.scalar(select(UserPolicyMirrorEvent)) + assert event.processed_at is not None + with v2_sessions() as session: + assert session.scalar(select(func.count()).select_from(UserPolicy)) == 1 + assert session.scalar(select(func.count()).select_from(User)) == 1 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 1 + ) + mapping = session.scalar(select(LegacyUserPolicyMapping)) + assert mapping.last_applied_source_revision == 1 + finally: + _cleanup(v2_engine, model_id) + v1_engine.dispose() + v2_engine.dispose() diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index a7cc3d79d..f1031637a 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -18,7 +18,10 @@ from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.data.v2.catalog.publication import publish_catalog -from policyengine_api.data.v2.catalog.query import V2MetadataQueryService +from policyengine_api.services.v2.metadata.database_session import ( + MetadataDatabaseSession, +) +from policyengine_api.services.v2.metadata.services import V2MetadataService from policyengine_api.data.v2.models import ( Dataset, TaxBenefitModel, @@ -78,8 +81,8 @@ def v1_metadata(country_id: str): gateway_client_factory=lambda: None, metadata_reader_factory=lambda: None, specification_provider=lambda: {}, - v2_metadata_reader_factory=lambda: V2MetadataQueryService( - Session(engine), + v2_metadata_reader_factory=lambda: V2MetadataService( + MetadataDatabaseSession(Session(engine)), running_policyengine_version=POLICYENGINE_VERSION, ), ) diff --git a/tests/integration/test_v2_policy_persistence.py b/tests/integration/test_v2_policy_persistence.py new file mode 100644 index 000000000..e5cad92b2 --- /dev/null +++ b/tests/integration/test_v2_policy_persistence.py @@ -0,0 +1,457 @@ +"""PostgreSQL transaction tests for immutable policy persistence.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import os +from threading import Barrier +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import create_engine, delete, func, select +from sqlalchemy.exc import IntegrityError +from sqlmodel import Session + +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.policies.transformations import ( + canonical_policy_document, + canonicalize_policy, +) +from policyengine_api.services.v2.policies.services import ( + create_resolved_policy, + mirror_legacy_policy_in_session, +) +from policyengine_api.services.v2.policies.types import ( + CanonicalPolicyContent, + LegacyPolicySnapshot, + ResolvedPolicyCreationInput, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, + PolicyContentHashCollisionError, +) +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get( + V2_ALEMBIC_DISPOSABLE_TEST, + "", + ), + } + ) + if not settings.disposable_test: + pytest.fail("policy persistence tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +def _catalog(session: Session, *, country_id: str | None = None): + unique = uuid4().hex + model = TaxBenefitModel( + name=( + f"policyengine-{country_id}" + if country_id is not None + else f"phase10-policy-{unique[:8]}" + ) + ) + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name=f"gov.phase10.{unique}", + tax_benefit_model_version=version, + ) + session.add(parameter) + session.flush() + return model, version, parameter + + +def _command( + model_id: UUID, + version_id: UUID, + parameter_id: UUID, + *, + value: object = 0.2, +) -> ResolvedPolicyCreationInput: + return ResolvedPolicyCreationInput( + country_id="us", + tax_benefit_model_id=model_id, + tax_benefit_model_version_id=version_id, + policyengine_version="5.2.0", + parameter_values=[ + { + "parameter_id": parameter_id, + "value": value, + "start_date": "2026-01-01T00:00:00Z", + } + ], + ) + + +def _cleanup(engine, model_id: UUID) -> None: + with engine.begin() as connection: + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + connection.execute( + delete(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id.in_(policy_ids) + ) + ) + connection.execute( + delete(ParameterValue).where(ParameterValue.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def test_equivalent_create_returns_one_policy_and_one_child_set() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session) + model_id = model.id + command = _command(model.id, version.id, parameter.id) + first = create_resolved_policy(session, command) + + with Session(engine) as session, session.begin(): + second = create_resolved_policy(session, command) + + assert first.created is True + assert second.created is False + assert second.policy_id == first.policy_id + + with Session(engine) as session: + policy_count = session.scalar( + select(func.count()) + .select_from(Policy) + .where(Policy.id == first.policy_id) + ) + value_count = session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id == first.policy_id) + ) + assert (policy_count, value_count) == (1, 1) + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_equal_hash_with_different_canonical_bytes_is_an_integrity_error() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session) + model_id = model.id + version_id = version.id + parameter_id = parameter.id + original = _command(model_id, version_id, parameter_id, value=0.2) + stored = canonicalize_policy(original) + create_resolved_policy(session, original) + + changed = _command(model_id, version_id, parameter_id, value=0.3) + + def simulated_collision( + command: ResolvedPolicyCreationInput, + ) -> CanonicalPolicyContent: + return CanonicalPolicyContent( + version=stored.version, + document=canonical_policy_document(command), + content_hash=stored.content_hash, + ) + + with Session(engine) as session, session.begin(): + with pytest.raises(PolicyContentHashCollisionError): + create_resolved_policy( + session, + changed, + canonicalizer=simulated_collision, + ) + + with Session(engine) as session: + values = session.scalars( + select(ParameterValue.value_json) + .join( + Policy, + Policy.id == ParameterValue.policy_id, + ) + .where(Policy.tax_benefit_model_id == model_id) + ).all() + assert values == [0.2] + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_legacy_policy_mapping_is_many_to_one_and_retry_safe() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session, country_id="us") + model_id = model.id + first = LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=301, + label="First label", + api_version="1.0.0", + policy_json={parameter.name: {"2026": 0.2}}, + source_policy_hash="first-legacy-hash", + ) + second = LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=302, + label="Second label", + api_version="1.0.0", + policy_json={parameter.name: {"2026": 0.2}}, + source_policy_hash="second-legacy-hash", + ) + first_result = mirror_legacy_policy_in_session( + session, + first, + running_policyengine_version=version.version, + country_package_versions={"us": "1.0.0"}, + ) + second_result = mirror_legacy_policy_in_session( + session, + second, + running_policyengine_version=version.version, + country_package_versions={"us": "1.0.0"}, + ) + + with Session(engine) as session, session.begin(): + retry = mirror_legacy_policy_in_session( + session, + first, + running_policyengine_version="5.2.0", + country_package_versions={"us": "1.0.0"}, + ) + + assert first_result.policy_id == second_result.policy_id == retry.policy_id + assert first_result.policy_created is True + assert second_result.policy_created is False + assert retry == type(retry)( + policy_id=first_result.policy_id, + policy_created=False, + mapping_created=False, + ) + with Session(engine) as session: + mappings = session.scalars( + select(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id == first_result.policy_id + ) + ).all() + assert {mapping.legacy_policy_id for mapping in mappings} == {301, 302} + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_changed_hash_for_one_legacy_identity_rolls_back_without_mutation() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session, country_id="us") + model_id = model.id + parameter_name = parameter.name + snapshot = LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=401, + api_version="1.0.0", + policy_json={parameter_name: {"2026": 0.2}}, + source_policy_hash="committed-source-hash", + ) + result = mirror_legacy_policy_in_session( + session, + snapshot, + running_policyengine_version=version.version, + country_package_versions={"us": "1.0.0"}, + ) + + changed = snapshot.model_copy( + update={ + "source_policy_hash": "different-source-hash", + "policy_json": {parameter_name: {"2026": 0.3}}, + } + ) + with pytest.raises(LegacyPolicyMappingIntegrityError, match="different"): + with Session(engine) as session, session.begin(): + mirror_legacy_policy_in_session( + session, + changed, + running_policyengine_version="5.2.0", + country_package_versions={"us": "1.0.0"}, + ) + + with Session(engine) as session: + mapping = session.scalar( + select(LegacyPolicyMapping).where( + LegacyPolicyMapping.legacy_policy_id == 401 + ) + ) + policy_count = session.scalar( + select(func.count()) + .select_from(Policy) + .where(Policy.tax_benefit_model_id == model_id) + ) + assert mapping is not None + assert mapping.policy_id == result.policy_id + assert mapping.source_policy_hash == "committed-source-hash" + assert policy_count == 1 + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_concurrent_equivalent_creates_return_one_policy_uuid() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session) + model_id = model.id + command = _command(model.id, version.id, parameter.id) + + barrier = Barrier(2, timeout=10) + + def create(): + with Session(engine) as session, session.begin(): + barrier.wait() + return create_resolved_policy(session, command) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(create) for _ in range(2)] + results = [future.result(timeout=15) for future in futures] + + assert {result.policy_id for result in results} == {results[0].policy_id} + assert sorted(result.created for result in results) == [False, True] + with Session(engine) as session: + policy_count = session.scalar( + select(func.count()) + .select_from(Policy) + .where(Policy.tax_benefit_model_id == model_id) + ) + value_count = session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id == results[0].policy_id) + ) + assert (policy_count, value_count) == (1, 1) + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_empty_and_distinct_policy_content_persist_independently() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, parameter = _catalog(session) + model_id = model.id + empty = ResolvedPolicyCreationInput( + country_id="us", + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=version.id, + policyengine_version=version.version, + parameter_values=[], + ) + first = create_resolved_policy(session, empty) + second = create_resolved_policy( + session, + _command(model.id, version.id, parameter.id, value=1), + ) + third = create_resolved_policy( + session, + _command(model.id, version.id, parameter.id, value=2), + ) + + assert len({first.policy_id, second.policy_id, third.policy_id}) == 3 + with Session(engine) as session: + empty_value_count = session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id == first.policy_id) + ) + assert empty_value_count == 0 + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() + + +def test_child_insert_failure_rolls_back_the_policy_and_all_values() -> None: + engine = create_engine(_disposable_url()) + model_id = None + try: + with Session(engine) as session, session.begin(): + model, version, _parameter = _catalog(session) + model_id = model.id + invalid = _command(model.id, version.id, uuid4()) + content_hash = canonicalize_policy(invalid).content_hash + + with pytest.raises(IntegrityError): + with Session(engine) as session, session.begin(): + create_resolved_policy(session, invalid) + + with Session(engine) as session: + policy_count = session.scalar( + select(func.count()) + .select_from(Policy) + .where(Policy.content_hash == content_hash) + ) + owned_value_count = session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id.is_not(None)) + ) + assert policy_count == 0 + assert owned_value_count == 0 + finally: + if model_id is not None: + _cleanup(engine, model_id) + engine.dispose() diff --git a/tests/integration/test_v2_user_policy_mirroring.py b/tests/integration/test_v2_user_policy_mirroring.py new file mode 100644 index 000000000..1b691451f --- /dev/null +++ b/tests/integration/test_v2_user_policy_mirroring.py @@ -0,0 +1,456 @@ +"""PostgreSQL transaction tests for legacy saved-policy projection.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import os +from threading import Barrier +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import create_engine, delete, func, select +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + LegacyUserMapping, + LegacyUserPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + User, + UserPolicy, +) +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.user_policies.services import ( + mirror_legacy_user_policy_in_session, + resolve_legacy_user_id, +) +from policyengine_api.services.v2.user_policies.transformations import ( + fingerprint_legacy_user_policy, +) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get( + V2_ALEMBIC_DISPOSABLE_TEST, + "", + ), + } + ) + if not settings.disposable_test: + pytest.fail("saved-policy mirror tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +def _seed_catalog(sessions): + with sessions.begin() as session: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version=POLICYENGINE_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.phase10.saved_policy_rate", + tax_benefit_model_version=version, + ) + session.add(parameter) + session.flush() + return model.id, parameter.name + + +def _reform(parameter_name: str, *, legacy_id: int, source_hash: str): + return LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=legacy_id, + label="Ignored legacy label", + api_version=COUNTRY_PACKAGE_VERSIONS["us"], + policy_json={parameter_name: {"2026": 0.2}}, + source_policy_hash=source_hash, + ) + + +def _saved(*, legacy_id: int, reform_id: int, reform_label="Reform", **changes): + values = { + "country_id": "us", + "legacy_user_policy_id": legacy_id, + "reform_id": reform_id, + "reform_label": reform_label, + "baseline_id": 1, + "baseline_label": "Current law", + "user_id": "auth0|one", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + values.update(changes) + return LegacyUserPolicySnapshot.model_validate(values) + + +def _cleanup(engine, model_id) -> None: + if model_id is None: + return + with engine.begin() as connection: + legacy_user_ids = ("auth0|one", "legacy-user-two") + user_ids = ( + connection.execute( + select(LegacyUserMapping.user_id).where( + LegacyUserMapping.legacy_user_id.in_(legacy_user_ids) + ) + ) + .scalars() + .all() + ) + connection.execute( + delete(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id.in_(legacy_user_ids) + ) + ) + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + association_ids = select(UserPolicy.id).where( + UserPolicy.policy_id.in_(policy_ids) + ) + connection.execute( + delete(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.user_policy_id.in_(association_ids) + ) + ) + connection.execute( + delete(UserPolicy).where(UserPolicy.policy_id.in_(policy_ids)) + ) + if user_ids: + connection.execute(delete(User).where(User.id.in_(user_ids))) + connection.execute( + delete(LegacyPolicyMapping).where( + LegacyPolicyMapping.policy_id.in_(policy_ids) + ) + ) + connection.execute( + delete(ParameterValue).where(ParameterValue.policy_id.in_(policy_ids)) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def test_saved_rows_share_policy_and_reuse_only_the_same_mapped_user() -> None: + engine = create_engine(_disposable_url()) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + model_id = None + try: + model_id, parameter_name = _seed_catalog(sessions) + first_reform = _reform(parameter_name, legacy_id=101, source_hash="first") + second_reform = _reform(parameter_name, legacy_id=102, source_hash="second") + first_saved = _saved(legacy_id=201, reform_id=101) + second_saved = _saved( + legacy_id=202, + reform_id=102, + reform_label=None, + ) + third_saved = _saved( + legacy_id=203, + reform_id=101, + reform_label="Other user", + user_id="legacy-user-two", + ) + + with sessions.begin() as session: + first = mirror_legacy_user_policy_in_session( + session, + first_saved, + first_reform, + source_revision=1, + ) + second = mirror_legacy_user_policy_in_session( + session, + second_saved, + second_reform, + source_revision=1, + ) + third = mirror_legacy_user_policy_in_session( + session, + third_saved, + first_reform, + source_revision=1, + ) + with sessions.begin() as session: + retry = mirror_legacy_user_policy_in_session( + session, + first_saved, + first_reform, + source_revision=1, + ) + + assert first.policy_id == second.policy_id + assert first.association_id != second.association_id + assert third.policy_id == first.policy_id + assert retry.association_id == first.association_id + assert retry.association_created is False + with sessions() as session: + associations = session.scalars( + select(UserPolicy).order_by(UserPolicy.created_at, UserPolicy.id) + ).all() + assert {association.name for association in associations} == { + "Reform", + None, + "Other user", + } + assert all(association.description is None for association in associations) + association_by_id = { + association.id: association for association in associations + } + assert association_by_id[first.association_id].user_id == ( + association_by_id[second.association_id].user_id + ) + assert association_by_id[third.association_id].user_id != ( + association_by_id[first.association_id].user_id + ) + assert session.scalar(select(func.count()).select_from(Policy)) == 1 + assert session.scalar(select(func.count()).select_from(User)) == 2 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 2 + ) + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 2 + ) + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 3 + ) + finally: + _cleanup(engine, model_id) + engine.dispose() + + +def test_concurrent_first_use_resolves_one_legacy_user_mapping() -> None: + engine = create_engine(_disposable_url()) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + legacy_user_id = f"phase10-concurrent-{uuid4()}" + barrier = Barrier(2) + + def resolve() -> UUID: + with sessions.begin() as session: + barrier.wait() + return resolve_legacy_user_id( + session, + legacy_user_id=legacy_user_id, + primary_country="us", + ) + + try: + with ThreadPoolExecutor(max_workers=2) as executor: + user_ids = list(executor.map(lambda _index: resolve(), range(2))) + + assert user_ids[0] == user_ids[1] + with sessions() as session: + mappings = session.scalars( + select(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == legacy_user_id + ) + ).all() + assert len(mappings) == 1 + assert session.get(User, user_ids[0]) is not None + finally: + with sessions.begin() as session: + mapping = session.get(LegacyUserMapping, legacy_user_id) + if mapping is not None: + user_id = mapping.user_id + session.delete(mapping) + session.flush() + user = session.get(User, user_id) + if user is not None: + session.delete(user) + engine.dispose() + + +def test_label_and_v1_only_updates_advance_the_complete_row_fingerprint() -> None: + engine = create_engine(_disposable_url()) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + model_id = None + try: + model_id, parameter_name = _seed_catalog(sessions) + reform = _reform(parameter_name, legacy_id=301, source_hash="reform") + original = _saved(legacy_id=401, reform_id=301) + with sessions.begin() as session: + created = mirror_legacy_user_policy_in_session( + session, + original, + reform, + source_revision=1, + ) + + renamed = original.model_copy( + update={"reform_label": "Renamed", "updated_date": 3} + ) + with sessions.begin() as session: + association = session.get(UserPolicy, created.association_id) + association.description = "Native description" + with sessions.begin() as session: + rename_result = mirror_legacy_user_policy_in_session( + session, + renamed, + reform, + source_revision=2, + changed_fields=frozenset({"reform_label", "updated_date"}), + ) + with sessions() as session: + after_rename = session.get(UserPolicy, created.association_id) + mapping = session.scalar( + select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.user_policy_id == created.association_id + ) + ) + assert after_rename.name == "Renamed" + assert after_rename.description == "Native description" + assert mapping.fingerprint_sha256 == fingerprint_legacy_user_policy(renamed) + + v1_only = renamed.model_copy(update={"year": "2027", "updated_date": 4}) + with sessions.begin() as session: + association = session.get(UserPolicy, created.association_id) + association.name = "Native name" + with sessions() as session: + native_timestamp = session.get( + UserPolicy, + created.association_id, + ).updated_at + with sessions.begin() as session: + v1_only_result = mirror_legacy_user_policy_in_session( + session, + v1_only, + reform, + source_revision=3, + changed_fields=frozenset({"year", "updated_date"}), + ) + with sessions() as session: + association = session.get(UserPolicy, created.association_id) + mapping = session.scalar( + select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.user_policy_id == created.association_id + ) + ) + + assert rename_result.association_updated is True + assert v1_only_result.association_updated is False + assert association.name == "Native name" + assert association.description == "Native description" + assert association.updated_at == native_timestamp + assert mapping.fingerprint_sha256 == fingerprint_legacy_user_policy(v1_only) + finally: + _cleanup(engine, model_id) + engine.dispose() + + +def test_complete_transaction_rolls_back_and_native_delete_is_isolated() -> None: + engine = create_engine(_disposable_url()) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + model_id = None + try: + model_id, parameter_name = _seed_catalog(sessions) + reform = _reform(parameter_name, legacy_id=501, source_hash="rollback") + saved = _saved(legacy_id=601, reform_id=501) + with pytest.raises(RuntimeError, match="forced rollback"): + with sessions.begin() as session: + mirror_legacy_user_policy_in_session( + session, + saved, + reform, + source_revision=1, + ) + raise RuntimeError("forced rollback") + + with sessions() as session: + assert session.scalar(select(func.count()).select_from(Policy)) == 0 + assert session.scalar(select(func.count()).select_from(UserPolicy)) == 0 + assert session.scalar(select(func.count()).select_from(User)) == 0 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 0 + ) + assert ( + session.scalar(select(func.count()).select_from(LegacyPolicyMapping)) + == 0 + ) + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 0 + ) + + with sessions.begin() as session: + created = mirror_legacy_user_policy_in_session( + session, + saved, + reform, + source_revision=1, + ) + with sessions.begin() as session: + association = session.get(UserPolicy, created.association_id) + session.delete(association) + + with sessions() as session: + assert session.get(UserPolicy, created.association_id) is None + assert ( + session.scalar( + select(func.count()).select_from(LegacyUserPolicyMapping) + ) + == 0 + ) + assert session.get(Policy, created.policy_id) is not None + assert session.scalar(select(func.count()).select_from(User)) == 1 + assert ( + session.scalar(select(func.count()).select_from(LegacyUserMapping)) == 1 + ) + assert ( + session.scalar( + select(func.count()) + .select_from(ParameterValue) + .where(ParameterValue.policy_id == created.policy_id) + ) + == 1 + ) + finally: + _cleanup(engine, model_id) + engine.dispose() diff --git a/tests/unit/data/test_v1_models.py b/tests/unit/data/test_v1_models.py index d61c5cacb..496e4adea 100644 --- a/tests/unit/data/test_v1_models.py +++ b/tests/unit/data/test_v1_models.py @@ -17,6 +17,7 @@ "simulation_runs", "simulations", "user_policies", + "user_policy_mirror_events", "user_profiles", } @@ -49,6 +50,14 @@ def test_v1_composite_and_unique_keys_match_legacy_contract(): "country_id", ] assert V1Base.metadata.tables["user_profiles"].c.auth0_id.unique + assert V1Base.metadata.tables["user_policies"].c.mirror_revision.default.arg == 0 + mirror_events = V1Base.metadata.tables["user_policy_mirror_events"] + assert mirror_events.c.id.autoincrement is True + assert { + tuple(column.name for column in constraint.columns) + for constraint in mirror_events.constraints + if constraint.__class__.__name__ == "UniqueConstraint" + } == {("country_id", "legacy_user_policy_id", "source_revision")} assert ( V1Base.metadata.tables[ "legacy_report_output_aliases" diff --git a/tests/unit/routes/test_migration_context_logging.py b/tests/unit/routes/test_migration_context_logging.py index 1a487dfeb..1aa6609fa 100644 --- a/tests/unit/routes/test_migration_context_logging.py +++ b/tests/unit/routes/test_migration_context_logging.py @@ -273,6 +273,50 @@ def test_v2_metadata_resource_logs_its_actual_supabase_read_source(monkeypatch): assert migration_context["db_read"] == "supabase" +def test_v2_policy_resources_log_actual_supabase_read_and_write_sources( + monkeypatch, +): + monkeypatch.setenv("DB_READ_POLICY", "invalid-for-native-v2") + monkeypatch.setenv("DB_WRITE_POLICY", "invalid-for-native-v2") + + with patch("policyengine_api.migration_logging.logger") as mock_logger: + log_migration_request( + request_id="request-read", + method="GET", + path="/v2/policies", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + log_migration_request( + request_id="request-write", + method="PATCH", + path="/v2/user-policies/00000000-0000-0000-0000-000000000001", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + + read_context = mock_logger.log_struct.call_args_list[0].args[0]["migration"] + write_context = mock_logger.log_struct.call_args_list[1].args[0]["migration"] + assert read_context == { + **read_context, + "route_group": "policy", + "route_impl": "fastapi_native", + "db_write": None, + "db_read": "supabase", + } + assert write_context == { + **write_context, + "route_group": "policy", + "route_impl": "fastapi_native", + "db_write": "supabase", + "db_read": None, + } + + def test_native_route_failure_logs_country_and_actual_implementation(): dependencies = NativeRouteDependencies( readiness_probe=lambda: True, diff --git a/tests/unit/routes/test_policy_dual_write_routes.py b/tests/unit/routes/test_policy_dual_write_routes.py new file mode 100644 index 000000000..39b2e1771 --- /dev/null +++ b/tests/unit/routes/test_policy_dual_write_routes.py @@ -0,0 +1,255 @@ +"""V1 policy route tests for immediate post-commit v2 mirroring.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +from flask import Flask +import pytest + +from policyengine_api.data.v1_models import Policy +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.routes.policy_routes import policy_bp +from policyengine_api.services.policy_mirroring import PolicyMirrorUnavailableError +from policyengine_api.services.policy_service import PolicySetResult + + +def _client(): + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(policy_bp) + return app.test_client() + + +def _snapshot(country_id: str = "us") -> LegacyPolicySnapshot: + return LegacyPolicySnapshot( + country_id=country_id, + legacy_policy_id=42, + label="Legacy label", + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + source_policy_hash="legacy/base64+hash=", + ) + + +def _creation( + *, + existing: bool = False, + country_id: str = "us", +) -> PolicySetResult: + return PolicySetResult( + policy_id=42, + message="Policy already exists" if existing else "Policy created", + is_existing_policy=existing, + snapshot=_snapshot(country_id), + ) + + +def _body() -> dict[str, object]: + return { + "label": "Legacy label", + "data": {"gov.example.rate": {"2026": 0.2}}, + } + + +def test_cloud_sql_mode_preserves_v1_create_response_without_mirroring( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy", + return_value=_creation(), + ) as set_policy, + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit" + ) as mirror, + ): + response = _client().post("/us/policy", json=_body()) + + assert response.status_code == 201 + assert response.json == { + "status": "ok", + "message": "Policy created", + "result": {"policy_id": 42}, + } + set_policy.assert_called_once_with( + "us", + "Legacy label", + {"gov.example.rate": {"2026": 0.2}}, + prepare_for_mirroring=False, + ) + mirror.assert_not_called() + + +@pytest.mark.parametrize("country_id", ("us", "uk")) +def test_dual_write_mirrors_new_and_existing_rows_before_success( + monkeypatch, + country_id, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + for existing, expected_status in ((False, 201), (True, 200)): + events: list[str] = [] + service = MagicMock( + side_effect=lambda *_args, **_kwargs: ( + events.append("cloud_sql") + or _creation(existing=existing, country_id=country_id) + ) + ) + mirror = MagicMock(side_effect=lambda _snapshot: events.append("supabase")) + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy", + service, + ), + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit", + mirror, + ), + ): + response = _client().post(f"/{country_id}/policy", json=_body()) + + assert response.status_code == expected_status + assert response.json["result"] == {"policy_id": 42} + assert "v2" not in response.json["result"] + assert events == ["cloud_sql", "supabase"] + assert service.call_args.kwargs == {"prepare_for_mirroring": True} + mirror.assert_called_once_with(_snapshot(country_id)) + + +@pytest.mark.parametrize("country_id", ("ca", "ng", "il")) +def test_dual_write_preserves_v1_only_policy_writes_for_non_v2_countries( + monkeypatch, + country_id, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + creation = PolicySetResult( + policy_id=42, + message="Policy created", + is_existing_policy=False, + snapshot=None, + ) + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy", + return_value=creation, + ) as set_policy, + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit" + ) as mirror, + ): + response = _client().post(f"/{country_id}/policy", json=_body()) + + assert response.status_code == 201 + assert response.json["result"] == {"policy_id": 42} + set_policy.assert_called_once_with( + country_id, + "Legacy label", + {"gov.example.rate": {"2026": 0.2}}, + prepare_for_mirroring=False, + ) + mirror.assert_not_called() + + +def test_mirror_failure_returns_503_and_identical_retry_completes( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + creation = _creation(existing=True) + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy", + return_value=creation, + ) as set_policy, + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit", + side_effect=[ + PolicyMirrorUnavailableError("database credential secret"), + MagicMock(), + ], + ) as mirror, + ): + first = _client().post("/us/policy", json=_body()) + retry = _client().post("/us/policy", json=_body()) + + assert first.status_code == 503 + assert first.json == { + "status": "error", + "message": "V2 policy mirroring is unavailable; retry the same request.", + } + assert "secret" not in first.text + assert retry.status_code == 200 + assert retry.json["result"] == {"policy_id": 42} + assert set_policy.call_count == 2 + assert mirror.call_count == 2 + + +def test_supabase_only_v1_write_selection_is_rejected_before_cloud_sql( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "supabase") + with patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy" + ) as set_policy: + response = _client().post("/us/policy", json=_body()) + + assert response.status_code == 503 + assert response.json["status"] == "error" + set_policy.assert_not_called() + + +def test_v1_policy_reads_require_cloud_sql_and_never_invoke_v2( + monkeypatch, +) -> None: + policy = Policy( + id=42, + country_id="us", + label="Legacy label", + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + policy_hash="legacy/base64+hash=", + ) + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + with ( + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy", + return_value=policy, + ) as get_policy, + patch( + "policyengine_api.routes.policy_routes.mirror_policy_after_commit" + ) as mirror, + ): + response = _client().get("/us/policy/42") + + assert response.status_code == 200 + assert json.loads(response.text)["result"]["id"] == 42 + get_policy.assert_called_once_with("us", 42) + mirror.assert_not_called() + + monkeypatch.setenv("DB_READ_POLICY", "read_compare") + with patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy" + ) as invalid_get: + invalid = _client().get("/us/policy/42") + + assert invalid.status_code == 503 + invalid_get.assert_not_called() + + +def test_policy_search_failure_does_not_expose_exception_details( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + with patch( + "policyengine_api.routes.policy_routes.policy_service.search_policies", + side_effect=RuntimeError("database-credential-secret"), + ): + response = _client().get("/us/policies?query=example") + + assert response.status_code == 500 + assert response.json == { + "status": "error", + "message": "Internal server error; please try again later.", + } + assert "database-credential-secret" not in response.text diff --git a/tests/unit/routes/test_user_policy_dual_write_routes.py b/tests/unit/routes/test_user_policy_dual_write_routes.py new file mode 100644 index 000000000..5b9d97734 --- /dev/null +++ b/tests/unit/routes/test_user_policy_dual_write_routes.py @@ -0,0 +1,534 @@ +"""V1 saved-policy route tests for immediate v2 association mirroring.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +from flask import Flask +import pytest +from sqlalchemy.exc import ( + IntegrityError, + OperationalError, + SQLAlchemyError, + TimeoutError as SQLAlchemyTimeoutError, +) + +from policyengine_api.data.v1_models import UserPolicy +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot +from policyengine_api.routes.policy_routes import policy_bp +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorUnavailableError, +) +from policyengine_api.services.user_policy_service import ( + UserPolicyCreateResult, + UserPolicyPersistenceError, + UserPolicyUpdateResult, +) + + +def _client(): + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(policy_bp) + return app.test_client() + + +def _row( + *, + country_id: str = "us", + reform_label: str | None = "Reform", + year: str = "2026", +): + return UserPolicy( + id=10, + country_id=country_id, + reform_id=2, + reform_label=reform_label, + baseline_id=1, + baseline_label="Current law", + user_id="auth0|one", + year=year, + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + + +def _snapshot( + *, + country_id: str = "us", + reform_label: str | None = "Reform", + year: str = "2026", +): + return LegacyUserPolicySnapshot( + country_id=country_id, + legacy_user_policy_id=10, + reform_id=2, + reform_label=reform_label, + baseline_id=1, + baseline_label="Current law", + user_id="auth0|one", + year=year, + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + + +def _reform_snapshot(country_id: str = "us"): + return LegacyPolicySnapshot( + country_id=country_id, + legacy_policy_id=2, + label="Ignored core label", + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + source_policy_hash="legacy/base64+hash=", + ) + + +def _creation( + *, + created=True, + country_id="us", + reform_label="Reform", + mirror_revision=1, +): + return UserPolicyCreateResult( + user_policy=_row(country_id=country_id, reform_label=reform_label), + created=created, + snapshot=_snapshot(country_id=country_id, reform_label=reform_label), + mirror_revision=mirror_revision, + ) + + +def _body(*, reform_label="Reform") -> dict[str, object]: + return { + "reform_id": 2, + "reform_label": reform_label, + "baseline_id": 1, + "baseline_label": "Current law", + "user_id": "auth0|one", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + + +def test_cloud_sql_mode_preserves_create_without_association_mirror( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.create_or_get_user_policy", + return_value=_creation(), + ), + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as mirror, + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy_snapshot" + ) as get_reform, + ): + response = _client().post("/us/user-policy", json=_body()) + + assert response.status_code == 201 + assert response.json["result"]["id"] == 10 + assert "v2" not in response.json["result"] + mirror.assert_not_called() + get_reform.assert_not_called() + + +@pytest.mark.parametrize("country_id", ("us", "uk")) +def test_dual_write_mirrors_new_existing_and_unlabeled_saved_policies( + monkeypatch, + country_id, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + for created, label, expected_status in ( + (True, "Reform", 201), + (False, "Reform", 200), + (True, None, 201), + ): + creation = _creation( + created=created, + country_id=country_id, + reform_label=label, + ) + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.create_or_get_user_policy", + return_value=creation, + ), + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy_snapshot", + return_value=_reform_snapshot(country_id), + ), + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as mirror, + ): + response = _client().post( + f"/{country_id}/user-policy", + json=_body(reform_label=label), + ) + + assert response.status_code == expected_status + assert response.json["result"]["id"] == 10 + assert "v2" not in response.json["result"] + mirror.assert_called_once() + assert mirror.call_args.args == (country_id, 10) + assert mirror.call_args.kwargs["through_revision"] == 1 + + +@pytest.mark.parametrize("country_id", ("ca", "ng", "il")) +def test_dual_write_preserves_v1_only_saved_policy_mutations_for_non_v2_countries( + monkeypatch, + country_id, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + creation = UserPolicyCreateResult( + user_policy=_row(country_id=country_id), + created=True, + snapshot=None, + mirror_revision=None, + ) + update = UserPolicyUpdateResult( + user_policy=_row(country_id=country_id, year="2027"), + snapshot=None, + changed_fields=frozenset({"year"}), + mirror_revision=None, + ) + + with ( + patch( + "policyengine_api.routes.policy_routes." + "user_policy_service.create_or_get_user_policy", + return_value=creation, + ) as create, + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as create_mirror, + ): + create_response = _client().post( + f"/{country_id}/user-policy", + json=_body(), + ) + + assert create_response.status_code == 201 + assert create.call_args.kwargs == {"record_mirror_event": False} + create_mirror.assert_not_called() + + with ( + patch( + "policyengine_api.routes.policy_routes." + "user_policy_service.update_user_policy", + return_value=update, + ) as update_saved_policy, + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as update_mirror, + ): + update_response = _client().put( + f"/{country_id}/user-policy", + json={"id": 10, "year": "2027"}, + ) + + assert update_response.status_code == 200 + update_saved_policy.assert_called_once_with( + country_id, + 10, + {"year": "2027"}, + record_mirror_event=False, + ) + update_mirror.assert_not_called() + + +def test_saved_policy_mirror_failure_returns_503_and_retry_completes( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + creation = _creation(created=False) + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.create_or_get_user_policy", + return_value=creation, + ) as create, + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy_snapshot", + return_value=_reform_snapshot(), + ), + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit", + side_effect=[ + UserPolicyMirrorUnavailableError("database credential secret"), + MagicMock(), + ], + ) as mirror, + ): + first = _client().post("/us/user-policy", json=_body()) + retry = _client().post("/us/user-policy", json=_body()) + + assert first.status_code == 503 + assert first.json == { + "message": "V2 saved-policy mirroring is unavailable; retry the same request." + } + assert "secret" not in first.text + assert retry.status_code == 200 + assert retry.json["result"] == {"id": 10} + assert create.call_count == mirror.call_count == 2 + + +@pytest.mark.parametrize( + ( + "method", + "service_method", + "body", + "error", + "expected_status", + "expected_category", + "expected_operation", + ), + ( + ( + "post", + "create_or_get_user_policy", + _body(), + SQLAlchemyTimeoutError("credential=timeout-secret"), + 503, + "timeout", + "create", + ), + ( + "put", + "update_user_policy", + {"id": 10, "reform_label": "Renamed"}, + OperationalError( + "UPDATE user_policies SET secret=:secret", + {"secret": "bound-parameter-secret"}, + RuntimeError("driver-secret"), + ), + 503, + "unavailable", + "update", + ), + ( + "post", + "create_or_get_user_policy", + _body(), + IntegrityError( + "INSERT caller-private-data", + {"user_id": "caller-private-data"}, + RuntimeError("integrity-secret"), + ), + 500, + "integrity", + "create", + ), + ( + "put", + "update_user_policy", + {"id": 10, "year": "2027"}, + SQLAlchemyError("database-secret"), + 500, + "database", + "update", + ), + ( + "post", + "create_or_get_user_policy", + _body(), + RuntimeError("unexpected-secret"), + 500, + "unexpected", + "create", + ), + ( + "put", + "update_user_policy", + {"id": "caller-id-secret", "year": "2027"}, + RuntimeError("unexpected-secret"), + 500, + "unexpected", + "update", + ), + ), +) +def test_saved_policy_persistence_failures_use_allowlisted_records( + monkeypatch, + method, + service_method, + body, + error, + expected_status, + expected_category, + expected_operation, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + with ( + patch( + f"policyengine_api.routes.policy_routes.user_policy_service.{service_method}", + side_effect=UserPolicyPersistenceError.from_exception(error), + ), + patch( + "policyengine_api.routes.policy_routes.current_request_id", + return_value="request-123", + ), + patch("policyengine_api.routes.policy_routes.logger.log_struct") as log_struct, + ): + response = getattr(_client(), method)("/us/user-policy", json=body) + + expected_message = ( + "Policy database is temporarily unavailable; please try again later." + if expected_status == 503 + else "Internal database error; please try again later." + ) + assert response.status_code == expected_status + assert response.json == {"message": expected_message} + + log_struct.assert_called_once() + assert log_struct.call_args.kwargs == {"severity": "ERROR"} + payload = log_struct.call_args.args[0] + assert set(payload) == { + "message", + "metric_name", + "metric_value", + "resource", + "operation", + "database_source", + "configured_write_source", + "country_id", + "legacy_user_policy_id", + "request_id", + "outcome", + "failure_category", + "http_status", + "duration_ms", + } + assert payload["operation"] == expected_operation + assert payload["failure_category"] == expected_category + assert payload["http_status"] == expected_status + assert payload["request_id"] == "request-123" + supplied_id = body.get("id") + expected_logged_id = ( + supplied_id + if isinstance(supplied_id, int) + and not isinstance(supplied_id, bool) + and 0 <= supplied_id <= 2_147_483_647 + else None + ) + assert payload["legacy_user_policy_id"] == expected_logged_id + + serialized_record = json.dumps(payload, sort_keys=True) + for private_value in ( + "timeout-secret", + "bound-parameter-secret", + "driver-secret", + "caller-private-data", + "integrity-secret", + "database-secret", + "unexpected-secret", + "caller-id-secret", + ): + assert private_value not in response.text + assert private_value not in serialized_record + + +def test_update_mirrors_projected_and_v1_only_changes_before_success( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + for payload, snapshot in ( + ({"reform_label": "Renamed"}, _snapshot(reform_label="Renamed")), + ({"year": "2027"}, _snapshot(year="2027")), + ): + update = UserPolicyUpdateResult( + user_policy=_row( + reform_label=snapshot.reform_label, + year=snapshot.year, + ), + snapshot=snapshot, + changed_fields=frozenset(payload), + mirror_revision=1, + ) + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.update_user_policy", + return_value=update, + ), + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy_snapshot", + return_value=_reform_snapshot(), + ), + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as mirror, + ): + response = _client().put( + "/us/user-policy", + json={"id": 10, **payload}, + ) + + assert response.status_code == 200 + assert response.json["result"] == {"id": 10} + assert mirror.call_args.args == ("us", 10) + assert mirror.call_args.kwargs["through_revision"] == 1 + + +def test_saved_policy_reads_remain_cloud_sql_only(monkeypatch) -> None: + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + with ( + patch( + "policyengine_api.routes.policy_routes.user_policy_service.list_user_policies", + return_value=[_row()], + ) as list_rows, + patch( + "policyengine_api.routes.policy_routes." + "mirror_pending_user_policy_events_after_commit" + ) as mirror, + ): + response = _client().get("/us/user-policy/auth0|one") + + assert response.status_code == 200 + assert response.json["result"][0]["id"] == 10 + list_rows.assert_called_once_with("us", "auth0|one") + mirror.assert_not_called() + + monkeypatch.setenv("DB_READ_POLICY", "supabase") + with patch( + "policyengine_api.routes.policy_routes.user_policy_service.list_user_policies" + ) as invalid_list: + invalid = _client().get("/us/user-policy/auth0|one") + + assert invalid.status_code == 503 + invalid_list.assert_not_called() + + +def test_v1_saved_policy_delete_is_not_added() -> None: + response = _client().delete("/us/user-policy", json={"id": 10}) + + assert response.status_code == 405 diff --git a/tests/unit/services/test_policy_mirroring.py b/tests/unit/services/test_policy_mirroring.py new file mode 100644 index 000000000..160b0afa5 --- /dev/null +++ b/tests/unit/services/test_policy_mirroring.py @@ -0,0 +1,105 @@ +"""Immediate v1 policy mirror orchestration and observability tests.""" + +from __future__ import annotations + +from uuid import UUID +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy.exc import OperationalError, TimeoutError + +from policyengine_api.services.v2.policies.types import ( + LegacyPolicyPersistenceResult, + LegacyPolicySnapshot, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyMappingIntegrityError, +) +from policyengine_api.services.policy_mirroring import ( + PolicyMirrorUnavailableError, + mirror_policy_after_commit, +) + + +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") + + +def _snapshot() -> LegacyPolicySnapshot: + return LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=42, + label="Legacy label", + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + source_policy_hash="legacy/base64+hash=", + ) + + +def test_success_returns_destination_and_logs_metric_without_policy_content() -> None: + mirror = MagicMock() + mirror.mirror_legacy_policy.return_value = LegacyPolicyPersistenceResult( + policy_id=POLICY_ID, + policy_created=True, + mapping_created=True, + ) + + with patch("policyengine_api.services.policy_mirroring.logger") as logger: + result = mirror_policy_after_commit( + _snapshot(), + mirror_factory=lambda: mirror, + ) + + assert result.policy_id == POLICY_ID + payload = logger.log_struct.call_args.args[0] + assert payload == { + **payload, + "metric_name": "v1_policy_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": ["cloud_sql", "supabase"], + "country_id": "us", + "legacy_policy_id": 42, + "destination_policy_id": str(POLICY_ID), + "outcome": "ok", + "failure_category": None, + "policy_created": True, + "mapping_created": True, + } + rendered = repr(payload) + assert "gov.example.rate" not in rendered + assert "legacy/base64" not in rendered + + +@pytest.mark.parametrize( + ("error", "category"), + [ + ( + OperationalError("statement secret", {}, Exception("credential")), + "database", + ), + (TimeoutError("pool timeout"), "database"), + (LegacyPolicyMappingIntegrityError("changed source hash"), "integrity"), + (RuntimeError("policy content secret"), "unexpected"), + ], +) +def test_failure_is_secret_safe_and_records_cloud_sql_as_only_completed_write( + error: Exception, + category: str, +) -> None: + mirror = MagicMock() + mirror.mirror_legacy_policy.side_effect = error + + with ( + patch("policyengine_api.services.policy_mirroring.logger") as logger, + pytest.raises(PolicyMirrorUnavailableError, match="could not be mirrored"), + ): + mirror_policy_after_commit(_snapshot(), mirror_factory=lambda: mirror) + + payload = logger.log_struct.call_args.args[0] + assert payload["outcome"] == "error" + assert payload["failure_category"] == category + assert payload["actual_write_sources"] == ["cloud_sql"] + assert payload["destination_policy_id"] is None + assert "secret" not in repr(payload) + assert "credential" not in repr(payload) diff --git a/tests/unit/services/test_policy_service.py b/tests/unit/services/test_policy_service.py index a2b1c8062..c8f008dd7 100644 --- a/tests/unit/services/test_policy_service.py +++ b/tests/unit/services/test_policy_service.py @@ -3,7 +3,7 @@ from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS from policyengine_api.data.v1_models import Policy -from policyengine_api.services.policy_service import PolicyService +from policyengine_api.services.policy_service import PolicyService, PolicySetResult from tests.fixtures.services.policy_service import valid_policy_data @@ -99,17 +99,28 @@ def test_set_policy_adds_mapped_entity(service, monkeypatch): lambda value: "new-hash", ) - policy_id, message, exists = service.set_policy( + result = service.set_policy( "US", "New policy", {"parameter": 1}, + prepare_for_mirroring=True, ) + policy_id, message, exists = result policy = service.get_policy("us", policy_id) assert policy.policy_json == {"parameter": 1} assert policy.api_version == COUNTRY_PACKAGE_VERSIONS["us"] assert message == "Policy created" assert exists is False + assert isinstance(result, PolicySetResult) + assert result.snapshot.model_dump() == { + "country_id": "us", + "legacy_policy_id": policy_id, + "label": "New policy", + "api_version": COUNTRY_PACKAGE_VERSIONS["us"], + "policy_json": {"parameter": 1}, + "source_policy_hash": "new-hash", + } def test_set_policy_returns_existing_mapped_entity( @@ -122,15 +133,38 @@ def test_set_policy_returns_existing_mapped_entity( lambda value: valid_policy_data["policy_hash"], ) - policy_id, message, exists = service.set_policy( + result = service.set_policy( "us", None, {}, + prepare_for_mirroring=True, ) + policy_id, message, exists = result assert policy_id == valid_policy_data["id"] assert message == "Policy already exists" assert exists is True + assert result.snapshot.legacy_policy_id == valid_policy_data["id"] + assert result.snapshot.source_policy_hash == valid_policy_data["policy_hash"] + + +def test_set_policy_does_not_build_v2_snapshot_unless_requested( + service, + monkeypatch, +): + monkeypatch.setattr( + "policyengine_api.services.policy_service.hash_object", + lambda value: "new-hash", + ) + + result = service.set_policy( + "ca", + "Canadian policy", + {"parameter": 1}, + ) + + assert result.snapshot is None + assert service.get_policy("ca", result.policy_id) is not None def test_set_policy_rejects_invalid_country(service): diff --git a/tests/unit/services/test_user_policy_mirroring.py b/tests/unit/services/test_user_policy_mirroring.py new file mode 100644 index 000000000..dc60d6fcf --- /dev/null +++ b/tests/unit/services/test_user_policy_mirroring.py @@ -0,0 +1,240 @@ +"""Saved-policy mirror observability and error conversion tests.""" + +from __future__ import annotations + +from uuid import UUID +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy import event as sqlalchemy_event, select +from sqlalchemy.exc import OperationalError, TimeoutError + +from policyengine_api.data.v1_models import UserPolicyMirrorEvent +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.validators import ( + LegacyUserPolicyIntegrityError, +) +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorUnavailableError, + mirror_pending_user_policy_events_after_commit, + mirror_user_policy_after_commit, +) +from policyengine_api.services.user_policy_service import UserPolicyService + + +ASSOCIATION_ID = UUID("00000000-0000-0000-0000-000000000060") +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") + + +def _saved() -> LegacyUserPolicySnapshot: + return LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=10, + reform_id=2, + reform_label="Reform", + baseline_id=1, + baseline_label="Current law", + user_id="auth0|one", + year="2026", + geography="us", + dataset="enhanced_cps_2024", + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + + +def _reform() -> LegacyPolicySnapshot: + return LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=2, + api_version="1.0.0", + policy_json={"gov.example.rate": {"2026": 0.2}}, + source_policy_hash="legacy/base64+hash=", + ) + + +def _saved_values() -> dict[str, object]: + return _saved().model_dump(exclude={"legacy_user_policy_id"}) + + +def test_success_logs_only_identifiers_outcomes_and_metric_fields() -> None: + mirror = MagicMock() + mirror.mirror_legacy_user_policy.return_value = LegacyUserPolicyPersistenceResult( + association_id=ASSOCIATION_ID, + policy_id=POLICY_ID, + association_created=True, + association_updated=False, + mapping_created=True, + ) + + with patch("policyengine_api.services.user_policy_mirroring.logger") as logger: + result = mirror_user_policy_after_commit( + _saved(), + _reform(), + source_revision=3, + changed_fields=frozenset({"reform_label"}), + mirror_factory=lambda: mirror, + ) + + assert result.association_id == ASSOCIATION_ID + mirror.mirror_legacy_user_policy.assert_called_once_with( + _saved(), + _reform(), + source_revision=3, + changed_fields=frozenset({"reform_label"}), + ) + payload = logger.log_struct.call_args.args[0] + assert payload["metric_name"] == "v1_user_policy_mirror_operations" + assert payload["configured_write_source"] == "dual_write" + assert payload["actual_write_sources"] == ["cloud_sql", "supabase"] + assert payload["legacy_user_policy_id"] == 10 + assert payload["destination_association_id"] == str(ASSOCIATION_ID) + assert payload["destination_policy_id"] == str(POLICY_ID) + rendered = repr(payload) + assert "gov.example.rate" not in rendered + assert "auth0|one" not in rendered + assert "Reform" not in rendered + + +@pytest.mark.parametrize( + ("error", "category"), + [ + ( + OperationalError("statement secret", {}, Exception("credential")), + "database", + ), + (TimeoutError("pool timeout"), "database"), + (LegacyUserPolicyIntegrityError("mapping conflict"), "integrity"), + (RuntimeError("caller data secret"), "unexpected"), + ], +) +def test_failure_is_secret_safe_and_reports_only_cloud_sql_completed( + error: Exception, + category: str, +) -> None: + mirror = MagicMock() + mirror.mirror_legacy_user_policy.side_effect = error + + with ( + patch("policyengine_api.services.user_policy_mirroring.logger") as logger, + pytest.raises(UserPolicyMirrorUnavailableError), + ): + mirror_user_policy_after_commit( + _saved(), + _reform(), + source_revision=3, + mirror_factory=lambda: mirror, + ) + + payload = logger.log_struct.call_args.args[0] + assert payload["actual_write_sources"] == ["cloud_sql"] + assert payload["failure_category"] == category + assert payload["destination_association_id"] is None + assert "secret" not in repr(payload) + assert "credential" not in repr(payload) + + +def test_processing_marker_commit_failure_logs_error_after_supabase_commit( + orm_session_factory, +) -> None: + event_service = UserPolicyService(orm_session_factory) + creation = event_service.create_or_get_user_policy( + _saved_values(), + record_mirror_event=True, + ) + mirror = MagicMock() + mirror.mirror_legacy_user_policy.return_value = LegacyUserPolicyPersistenceResult( + association_id=ASSOCIATION_ID, + policy_id=POLICY_ID, + association_created=True, + association_updated=False, + mapping_created=True, + ) + source_commit_error = OperationalError( + "processed_at update secret", + {"caller": "caller data secret"}, + Exception("database credential"), + ) + + def fail_source_commit(_session) -> None: + raise source_commit_error + + sqlalchemy_event.listen( + orm_session_factory.class_, + "before_commit", + fail_source_commit, + ) + try: + with ( + patch("policyengine_api.services.user_policy_mirroring.logger") as logger, + pytest.raises(UserPolicyMirrorUnavailableError), + ): + mirror_pending_user_policy_events_after_commit( + "us", + creation.user_policy.id, + through_revision=creation.mirror_revision, + event_service=event_service, + reform_snapshot_loader=lambda _country_id, _policy_id: _reform(), + mirror_factory=lambda: mirror, + ) + finally: + sqlalchemy_event.remove( + orm_session_factory.class_, + "before_commit", + fail_source_commit, + ) + + logger.log_struct.assert_called_once() + payload = logger.log_struct.call_args.args[0] + assert payload["outcome"] == "error" + assert payload["failure_category"] == "database" + assert payload["actual_write_sources"] == ["cloud_sql", "supabase"] + assert payload["destination_association_id"] == str(ASSOCIATION_ID) + assert payload["destination_policy_id"] == str(POLICY_ID) + assert "secret" not in repr(payload) + assert "credential" not in repr(payload) + with orm_session_factory() as session: + retained_event = session.scalar(select(UserPolicyMirrorEvent)) + assert retained_event.processed_at is None + + +def test_event_preparation_failure_logs_error_and_retains_the_event( + orm_session_factory, +) -> None: + event_service = UserPolicyService(orm_session_factory) + creation = event_service.create_or_get_user_policy( + _saved_values(), + record_mirror_event=True, + ) + mirror_factory = MagicMock() + + with ( + patch("policyengine_api.services.user_policy_mirroring.logger") as logger, + pytest.raises(UserPolicyMirrorUnavailableError), + ): + mirror_pending_user_policy_events_after_commit( + "us", + creation.user_policy.id, + through_revision=creation.mirror_revision, + event_service=event_service, + reform_snapshot_loader=lambda _country_id, _policy_id: None, + mirror_factory=mirror_factory, + ) + + mirror_factory.assert_not_called() + logger.log_struct.assert_called_once() + payload = logger.log_struct.call_args.args[0] + assert payload["outcome"] == "error" + assert payload["actual_write_sources"] == ["cloud_sql"] + assert payload["source_revision"] == creation.mirror_revision + with orm_session_factory() as session: + retained_event = session.scalar(select(UserPolicyMirrorEvent)) + assert retained_event.processed_at is None diff --git a/tests/unit/services/test_user_policy_service.py b/tests/unit/services/test_user_policy_service.py index 44a96f363..7fec2014e 100644 --- a/tests/unit/services/test_user_policy_service.py +++ b/tests/unit/services/test_user_policy_service.py @@ -1,16 +1,35 @@ import inspect from pathlib import Path +from unittest.mock import MagicMock, patch +from uuid import UUID -from policyengine_api.data.v1_models import UserPolicy +import pytest +from sqlalchemy import event as sqlalchemy_event +from sqlalchemy import func, select +from sqlalchemy.exc import ( + IntegrityError, + OperationalError, + SQLAlchemyError, + TimeoutError as SQLAlchemyTimeoutError, +) + +from policyengine_api.data.v1_models import UserPolicy, UserPolicyMirrorEvent +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyPersistenceResult, +) from policyengine_api.services.user_policy_service import ( UserPolicyCreateResult, + UserPolicyMirrorEventIntegrityError, + UserPolicyPersistenceError, UserPolicyService, + UserPolicyUpdateResult, ) ROUTE_PATH = ( Path(__file__).parents[3] / "policyengine_api" / "routes" / "policy_routes.py" ) +APPLICATION_ROOT = Path(__file__).parents[3] / "policyengine_api" def _values(**overrides): @@ -55,6 +74,79 @@ def test_policy_routes_do_not_manage_sessions_or_queries(): assert "select(" not in source +@pytest.mark.parametrize( + ("error", "expected_category", "expected_retryable"), + ( + ( + SQLAlchemyTimeoutError("credential=timeout-secret"), + "timeout", + True, + ), + ( + OperationalError( + "UPDATE private_table SET token=:token", + {"token": "bound-parameter-secret"}, + RuntimeError("driver-secret"), + ), + "unavailable", + True, + ), + ( + IntegrityError( + "INSERT caller-private-data", + {"user_id": "caller-private-data"}, + RuntimeError("integrity-secret"), + ), + "integrity", + False, + ), + (SQLAlchemyError("database-secret"), "database", False), + (RuntimeError("unexpected-secret"), "unexpected", False), + ), +) +def test_saved_policy_service_translates_failures_to_safe_domain_errors( + error, + expected_category, + expected_retryable, +) -> None: + session_factory = MagicMock() + session_factory.begin.side_effect = error + service = UserPolicyService(session_factory) + + with pytest.raises(UserPolicyPersistenceError) as raised: + service.create_or_get_user_policy(_values()) + + assert raised.value.category == expected_category + assert raised.value.retryable is expected_retryable + assert str(raised.value) == "Saved-policy persistence failed" + + +def test_saved_policy_persistence_failure_category_is_allowlisted() -> None: + with pytest.raises(ValueError): + UserPolicyPersistenceError("raw-exception-text") # type: ignore[arg-type] + + +def test_saved_policy_event_processing_has_only_request_path_callers(): + sources = { + path.relative_to(APPLICATION_ROOT).as_posix(): path.read_text(encoding="utf-8") + for path in APPLICATION_ROOT.rglob("*.py") + } + + assert { + path + for path, source in sources.items() + if ".process_pending_mirror_events(" in source + } == {"services/user_policy_mirroring.py"} + assert { + path + for path, source in sources.items() + if "mirror_pending_user_policy_events_after_commit(" in source + } == { + "routes/policy_routes.py", + "services/user_policy_mirroring.py", + } + + def test_create_reuse_list_and_update_user_policy(orm_session_factory): service = UserPolicyService(orm_session_factory) @@ -73,10 +165,34 @@ def test_create_reuse_list_and_update_user_policy(orm_session_factory): assert created.created is True assert reused.created is False assert reused.user_policy.id == created.user_policy.id + assert created.snapshot is None + assert reused.snapshot is None assert len(listed) == 1 assert isinstance(listed[0], UserPolicy) - assert updated.reform_label == "Updated" - assert updated.updated_date == 3 + assert isinstance(updated, UserPolicyUpdateResult) + assert updated.user_policy.reform_label == "Updated" + assert updated.user_policy.updated_date == 3 + assert updated.snapshot is None + assert updated.changed_fields == frozenset({"reform_label", "updated_date"}) + + +def test_v1_only_saved_policy_mutations_do_not_build_v2_snapshots( + orm_session_factory, +): + service = UserPolicyService(orm_session_factory) + + created = service.create_or_get_user_policy(_values(country_id="ca")) + updated = service.update_user_policy( + "ca", + created.user_policy.id, + {"reform_label": "Canadian policy"}, + ) + + assert created.snapshot is None + assert updated is not None + assert updated.snapshot is None + with orm_session_factory() as session: + assert session.scalar(select(func.count(UserPolicyMirrorEvent.id))) == 0 def test_update_user_policy_requires_matching_country(orm_session_factory): @@ -93,3 +209,257 @@ def test_update_user_policy_requires_matching_country(orm_session_factory): ) stored = service.list_user_policies("uk", "auth0|one")[0] assert stored.reform_label == "Reform" + + +def test_dual_write_mutations_store_ordered_complete_events_atomically( + orm_session_factory, +): + service = UserPolicyService(orm_session_factory) + + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + updated = service.update_user_policy( + "us", + created.user_policy.id, + {"reform_label": None, "updated_date": 3}, + record_mirror_event=True, + ) + + assert created.mirror_revision == 1 + assert created.snapshot is not None + assert updated is not None + assert updated.mirror_revision == 2 + assert updated.snapshot is not None + assert updated.snapshot.reform_label is None + assert updated.snapshot.updated_date == 3 + assert updated.snapshot.legacy_user_policy_id == created.user_policy.id + with orm_session_factory() as session: + stored = session.get(UserPolicy, created.user_policy.id) + events = session.scalars( + select(UserPolicyMirrorEvent).order_by( + UserPolicyMirrorEvent.source_revision + ) + ).all() + + assert stored.mirror_revision == 2 + assert [event.source_revision for event in events] == [1, 2] + assert [event.event_type for event in events] == ["create", "update"] + assert events[0].payload_json["changed_fields"] == [] + assert events[1].payload_json["changed_fields"] == [ + "reform_label", + "updated_date", + ] + assert events[1].payload_json["snapshot"]["reform_label"] is None + assert all(len(event.source_fingerprint_sha256) == 64 for event in events) + assert all(event.processed_at is None for event in events) + + +def test_event_failure_rolls_back_the_source_mutation(orm_session_factory): + service = UserPolicyService(orm_session_factory) + + with ( + patch( + "policyengine_api.services.user_policy_service." + "fingerprint_legacy_user_policy", + side_effect=RuntimeError("cannot encode event"), + ), + pytest.raises(UserPolicyPersistenceError) as raised, + ): + service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + + assert raised.value.category == "unexpected" + assert isinstance(raised.value.__cause__, RuntimeError) + with orm_session_factory() as session: + assert session.scalar(select(func.count()).select_from(UserPolicy)) == 0 + assert ( + session.scalar(select(func.count()).select_from(UserPolicyMirrorEvent)) == 0 + ) + + +def test_update_event_failure_rolls_back_the_source_update(orm_session_factory): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy(_values()) + + with ( + patch( + "policyengine_api.services.user_policy_service." + "fingerprint_legacy_user_policy", + side_effect=RuntimeError("cannot encode event"), + ), + pytest.raises(UserPolicyPersistenceError) as raised, + ): + service.update_user_policy( + "us", + created.user_policy.id, + {"reform_label": "Should roll back"}, + record_mirror_event=True, + ) + + assert raised.value.category == "unexpected" + assert isinstance(raised.value.__cause__, RuntimeError) + with orm_session_factory() as session: + stored = session.get(UserPolicy, created.user_policy.id) + assert stored.reform_label == "Reform" + assert stored.mirror_revision == 0 + assert ( + session.scalar(select(func.count()).select_from(UserPolicyMirrorEvent)) == 0 + ) + + +def test_pending_events_process_in_revision_order_and_mark_after_success( + orm_session_factory, +): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + updated = service.update_user_policy( + "us", + created.user_policy.id, + {"reform_label": "Second"}, + record_mirror_event=True, + ) + processed = [] + committed = [] + + def process(event): + processed.append((event.source_revision, event.changed_fields)) + return LegacyUserPolicyPersistenceResult( + association_id=UUID("00000000-0000-0000-0000-000000000060"), + policy_id=UUID("00000000-0000-0000-0000-000000000010"), + association_created=event.source_revision == 1, + association_updated=event.source_revision == 2, + mapping_created=event.source_revision == 1, + ) + + def record_source_commit(_session): + committed.append(("source_commit", len(processed))) + + def record_processed_commit(event, _result): + committed.append(("completion_callback", event.source_revision)) + + sqlalchemy_event.listen( + orm_session_factory.class_, + "after_commit", + record_source_commit, + ) + try: + result = service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=updated.mirror_revision, + processor=process, + after_processed_commit=record_processed_commit, + ) + finally: + sqlalchemy_event.remove( + orm_session_factory.class_, + "after_commit", + record_source_commit, + ) + + assert processed == [ + (1, frozenset()), + (2, frozenset({"reform_label"})), + ] + assert committed == [ + ("source_commit", 1), + ("completion_callback", 1), + ("source_commit", 2), + ("completion_callback", 2), + ] + assert result.association_updated is True + with orm_session_factory() as session: + events = session.scalars(select(UserPolicyMirrorEvent)).all() + assert all(event.processed_at is not None for event in events) + + +def test_failed_processing_retains_the_oldest_pending_event(orm_session_factory): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + + with pytest.raises(RuntimeError, match="Supabase unavailable"): + service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + processor=lambda event: (_ for _ in ()).throw( + RuntimeError("Supabase unavailable") + ), + ) + + with orm_session_factory() as session: + event = session.scalar(select(UserPolicyMirrorEvent)) + assert event.processed_at is None + + +def test_already_processed_request_revision_is_verified_by_idempotent_replay( + orm_session_factory, +): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + calls = [] + + def process(event): + calls.append(event.source_revision) + return LegacyUserPolicyPersistenceResult( + association_id=UUID("00000000-0000-0000-0000-000000000060"), + policy_id=UUID("00000000-0000-0000-0000-000000000010"), + association_created=len(calls) == 1, + association_updated=False, + mapping_created=len(calls) == 1, + ) + + service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + processor=process, + ) + replay = service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + processor=process, + ) + + assert calls == [1, 1] + assert replay.association_created is False + + +def test_corrupt_event_changed_fields_are_rejected(orm_session_factory): + service = UserPolicyService(orm_session_factory) + created = service.create_or_get_user_policy( + _values(), + record_mirror_event=True, + ) + with orm_session_factory.begin() as session: + event = session.scalar(select(UserPolicyMirrorEvent)) + event.event_type = "update" + event.payload_json = { + **event.payload_json, + "changed_fields": ["not_a_mutable_source_field"], + } + + with pytest.raises( + UserPolicyMirrorEventIntegrityError, + match="changed fields are unsupported", + ): + service.process_pending_mirror_events( + "us", + created.user_policy.id, + through_revision=created.mirror_revision, + processor=lambda event: None, + ) diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index b996c79f1..c624c686f 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -84,7 +84,7 @@ def test_push_requires_schema_and_v2_integration_checks_before_versioning(): versioning_job = workflow[workflow.index(" versioning:") :] versioning_job = versioning_job[: versioning_job.index("\n publish-git-tag:")] tag_job = workflow[workflow.index(" publish-git-tag:") :] - tag_job = tag_job[: tag_job.index("\n migrate-v1-cloud-sql:")] + tag_job = tag_job[: tag_job.index("\n migrate-v1-staging-cloud-sql:")] assert "lint:" in workflow assert "alembic-v1-check:" in workflow @@ -106,16 +106,12 @@ def test_push_requires_schema_and_v2_integration_checks_before_versioning(): def test_release_migration_uses_the_installed_python_environment(): - workflow = _workflow("push.yml") - migration_job = workflow[workflow.index(" migrate-v1-cloud-sql:") :] - migration_job = migration_job[ - : migration_job.index("\n deploy-cloud-run-staging:") - ] + workflow = _workflow("migrate-v1-cloud-sql.yml") orchestration_script = ( REPO / ".github" / "scripts" / "migrate_v1_cloud_sql.sh" ).read_text(encoding="utf-8") - assert "bash .github/scripts/migrate_v1_cloud_sql.sh" in migration_job + assert "bash .github/scripts/migrate_v1_cloud_sql.sh" in workflow assert "python scripts/v1_database_migration.py" in orchestration_script assert "uv run" not in orchestration_script @@ -180,12 +176,16 @@ def test_reusable_v2_integration_check_uses_postgres_redis_and_coverage(): assert "RUN_V2_CATALOG_COMPATIBILITY" in workflow assert "test_v2_catalog_publication.py" in workflow assert "test_v2_metadata_routes.py" in workflow + assert "test_v2_policy_persistence.py" in workflow + assert "test_v1_policy_dual_write.py" in workflow + assert "test_v2_user_policy_mirroring.py" in workflow + assert "test_v1_user_policy_dual_write.py" in workflow assert "test_v2_catalog_publication_qualification.py" in workflow assert "RUN_V2_CATALOG_PUBLICATION_QUALIFICATION" in workflow assert "test_runtime_cache_redis.py" in workflow assert "uv sync --frozen" in workflow assert workflow.count("coverage run --branch") == 1 - assert workflow.count("coverage run -a --branch") == 3 + assert workflow.count("coverage run -a --branch") == 4 assert "coverage xml -i -o coverage-v2.xml" in workflow assert "codecov/codecov-action@v5" in workflow assert "files: coverage-v2.xml" in workflow @@ -197,8 +197,8 @@ def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy(): REPO / ".github" / "scripts" / "migrate_v1_cloud_sql.sh" ).read_text(encoding="utf-8") - assert "migrate-v1-cloud-sql:" in workflow - assert "environment: production-database" in workflow + assert "migrate-v1-staging-cloud-sql:" in workflow + assert "migrate-v1-production-cloud-sql:" in workflow assert "--mode state" in orchestration_script assert "--mode upgrade" in orchestration_script assert "--mode verify-head" in orchestration_script @@ -209,7 +209,7 @@ def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy(): cloud_run_job = cloud_run_job[ : cloud_run_job.index("\n integration-tests-staging-cloud-run:") ] - assert "migrate-v1-cloud-sql" in cloud_run_job + assert "migrate-v1-staging-cloud-sql" in cloud_run_job assert "make test" in cloud_run_job assert cloud_run_job.index("make test") < cloud_run_job.index( 'uses: "google-github-actions/auth@v2"' @@ -220,11 +220,7 @@ def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy(): def test_cloud_sql_workflow_uses_oidc_and_separate_database_credentials(): - workflow = _workflow("push.yml") - migration_job = workflow[workflow.index(" migrate-v1-cloud-sql:") :] - migration_job = migration_job[ - : migration_job.index("\n deploy-cloud-run-staging:") - ] + migration_job = _workflow("migrate-v1-cloud-sql.yml") orchestration_script = ( REPO / ".github" / "scripts" / "migrate_v1_cloud_sql.sh" ).read_text(encoding="utf-8") @@ -242,6 +238,37 @@ def test_cloud_sql_workflow_uses_oidc_and_separate_database_credentials(): ) +def test_database_environment_validation_rejects_production_migration_credentials_in_staging(): + result = subprocess.run( + ["bash", ".github/scripts/validate_database_environment.sh", "cloud-sql"], + cwd=REPO, + env={ + **os.environ, + "DEPLOYMENT_ENVIRONMENT": "staging", + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + "project:region:staging-instance" + ), + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + "project:region:production-instance" + ), + "POLICYENGINE_DB_READONLY_PASSWORD_SECRET": "production-readonly", + "POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": "staging-migration", + "PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "production-readonly" + ), + "PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "production-migration" + ), + }, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 1 + assert "must be distinct from production" in result.stderr + + def _write_fake_migration_commands(tmp_path: Path) -> tuple[Path, Path]: bin_path = tmp_path / "bin" bin_path.mkdir() @@ -288,6 +315,22 @@ def _run_migration_orchestrator(tmp_path: Path, database_state: str): "GCLOUD_CALLS": str(gcloud_calls), "PYTHON_CALLS": str(python_calls), "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": "project:region:instance", + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + "project:region:instance" + ), + "DEPLOYMENT_ENVIRONMENT": "production", + "POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "policyengine-api-prod-db-readonly-password" + ), + "POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "policyengine-api-prod-db-migration-password" + ), + "PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "policyengine-api-prod-db-readonly-password" + ), + "PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "policyengine-api-prod-db-migration-password" + ), "PATH": f"{bin_path}:{os.environ['PATH']}", }, capture_output=True, diff --git a/tests/unit/test_app_engine_decommission.py b/tests/unit/test_app_engine_decommission.py index 7d738f38d..1a5896f0f 100644 --- a/tests/unit/test_app_engine_decommission.py +++ b/tests/unit/test_app_engine_decommission.py @@ -69,6 +69,7 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: "integration-tests-staging-cloud-run", ) staging_promotion = _job_block(workflow, "promote-cloud-run-staging") + staging_phase10_exercise = _job_block(workflow, "exercise-phase10-staging") production_check = _job_block( workflow, "ensure-production-model-version-aligns-with-sim-api", @@ -78,7 +79,7 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: docker_publish = _job_block(workflow, "docker") assert " release-tests:" not in workflow - assert "migrate-v1-cloud-sql" in staging_seed + assert "migrate-v1-staging-cloud-sql" in staging_seed assert "- seed-v2-staging-database" in staging_deploy assert "make test" in staging_deploy assert staging_deploy.index("make test") < staging_deploy.index( @@ -89,10 +90,9 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: ) assert "needs: deploy-cloud-run-staging" in staging_integration assert "- integration-tests-staging-cloud-run" in staging_promotion - assert "needs: promote-cloud-run-staging" in production_check - assert ( - "needs: ensure-production-model-version-aligns-with-sim-api" in production_seed - ) + assert "- promote-cloud-run-staging" in staging_phase10_exercise + assert "needs: exercise-phase10-staging" in production_check + assert "migrate-v1-production-cloud-sql" in production_seed assert "needs: seed-v2-production-database" in production_deploy assert "needs: deploy-cloud-run-candidate" in docker_publish diff --git a/tests/unit/test_asgi_factory.py b/tests/unit/test_asgi_factory.py index 92d2649ce..997eb8e04 100644 --- a/tests/unit/test_asgi_factory.py +++ b/tests/unit/test_asgi_factory.py @@ -3,6 +3,7 @@ import sys import threading from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace from types import SimpleNamespace from unittest.mock import Mock, patch @@ -11,7 +12,7 @@ from fastapi.testclient import TestClient from flask import Flask, Response, jsonify, make_response, request from flask_cors import CORS -from policyengine_api.asgi_factory import _add_vary_origin, create_asgi_app +from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, @@ -20,7 +21,6 @@ REQUEST_ID_HEADER, current_request_id, ) -from starlette.responses import Response as ASGIResponse def create_test_wsgi_app() -> Flask: @@ -367,25 +367,6 @@ def make_request(request_id): } -@pytest.mark.parametrize( - ("existing_vary", "expected_vary"), - [ - (None, "Origin"), - ("Accept-Encoding", "Accept-Encoding, Origin"), - ("Origin", "Origin"), - ("Accept-Encoding, origin", "Accept-Encoding, origin"), - ], -) -def test_add_vary_origin_preserves_existing_values(existing_vary, expected_vary): - response = ASGIResponse() - if existing_vary is not None: - response.headers["Vary"] = existing_vary - - _add_vary_origin(response) - - assert response.headers["Vary"] == expected_vary - - def test_asgi_entrypoint_imports_and_serves_health(monkeypatch): monkeypatch.setenv("FLASK_DEBUG", "1") sys.modules.pop("policyengine_api.asgi", None) @@ -498,6 +479,181 @@ def test_health_route_uses_same_reflected_cors_policy(): assert response.headers["vary"] == "Origin" +@pytest.mark.parametrize( + ("path", "method"), + [ + ("/v2/policies?country_id=us", "POST"), + ( + "/v2/user-policies/00000000-0000-0000-0000-000000000001?country_id=us", + "PATCH", + ), + ( + "/v2/user-policies/00000000-0000-0000-0000-000000000001?country_id=us", + "DELETE", + ), + ], +) +def test_cors_preflight_is_handled_before_v2_route_resolution(path, method): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.options( + path, + headers={ + "Origin": "https://app.policyengine.org", + "Access-Control-Request-Method": method, + "Access-Control-Request-Headers": ( + "authorization, content-type, x-policyengine-request-id" + ), + }, + ) + + assert response.status_code == 200 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + assert method in response.headers["access-control-allow-methods"].split(", ") + assert response.headers["access-control-allow-headers"] == ( + "authorization, content-type, x-policyengine-request-id" + ) + assert response.headers["access-control-max-age"] == "600" + assert response.headers["vary"] == "Origin" + + +def test_cors_preflight_rejects_a_method_outside_the_public_http_contract(): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.options( + "/v2/policies?country_id=us", + headers={ + "Origin": "https://app.policyengine.org", + "Access-Control-Request-Method": "TRACE", + }, + ) + + assert response.status_code == 400 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + + +def test_cors_preflight_does_not_invoke_the_mounted_flask_application(): + def failing_wsgi_app(_environ, _start_response): + raise AssertionError("preflight reached Flask") + + client = TestClient(create_asgi_app(failing_wsgi_app)) + + response = client.options( + "/fallback", + headers={ + "Origin": "https://app.policyengine.org", + "Access-Control-Request-Method": "GET", + }, + ) + + assert response.status_code == 200 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + + +def test_non_cors_options_request_uses_normal_v2_method_routing(): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.options("/v2/policies?country_id=us") + + assert response.status_code == 405 + assert response.json() == { + "status": "error", + "message": "API v2 resource 'policies' does not support this method", + } + + +@pytest.mark.parametrize( + ("method", "path", "expected_status"), + [ + ("GET", "/v2/policies", 422), + ("GET", "/v2/not-a-resource", 404), + ("OPTIONS", "/v2/policies?country_id=us", 405), + ], +) +def test_cors_headers_are_present_on_v2_error_responses( + method, + path, + expected_status, +): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.request( + method, + path, + headers={"Origin": "https://app.policyengine.org"}, + ) + + assert response.status_code == expected_status + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + assert REQUEST_ID_HEADER.lower() in { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + + +def test_cors_exposes_response_request_id_to_browser_clients(): + client = TestClient(create_asgi_app(create_test_wsgi_app())) + + response = client.get( + "/health", + headers={ + "Origin": "https://app.policyengine.org", + REQUEST_ID_HEADER: "browser-request-id", + }, + ) + + assert response.headers[REQUEST_ID_HEADER] == "browser-request-id" + exposed_headers = { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + assert REQUEST_ID_HEADER.lower() in exposed_headers + + +def test_cors_headers_are_present_on_unhandled_native_errors(): + def raise_unhandled_error() -> bool: + raise RuntimeError("unavailable") + + dependencies = replace( + _stage6_dependencies(), + readiness_probe=raise_unhandled_error, + ) + client = TestClient( + create_asgi_app( + create_test_wsgi_app(), + route_settings=_stage6_settings(health=RouteImplementation.FASTAPI_NATIVE), + dependencies=dependencies, + ), + raise_server_exceptions=False, + ) + response = client.get( + "/readiness-check", + headers={ + "Origin": "https://app.policyengine.org", + REQUEST_ID_HEADER: "failed-request-id", + }, + ) + + assert response.status_code == 500 + assert response.headers[REQUEST_ID_HEADER] == "failed-request-id" + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + exposed_headers = { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + assert REQUEST_ID_HEADER.lower() in exposed_headers + + def test_public_simulation_gateway_health_probe_checks_gateway(): client = TestClient(create_asgi_app(create_test_wsgi_app())) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index d6486030a..31e229c47 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -74,7 +74,21 @@ def _v2_target_env() -> dict[str, str]: def _required_runtime_env() -> dict[str, str]: return { + "DEPLOYMENT_ENVIRONMENT": "production", "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": PRODUCTION_CLOUD_SQL_INSTANCE, + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + PRODUCTION_CLOUD_SQL_INSTANCE + ), + "PRODUCTION_V2_SUPABASE_PROJECT_REF": TEST_V2_PROJECT_REF, + "PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": ( + TEST_V2_RUNTIME_SECRET_RESOURCE + ), + "PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET": ( + "policyengine-api-prod-db-password:latest" + ), + "CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET": ( + "policyengine-api-prod-db-password:latest" + ), "POLICYENGINE_DB_PASSWORD": "raw-db-secret-value", "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": ("raw-github-secret-value"), "OPENAI_API_KEY": "raw-openai-secret-value", @@ -85,6 +99,9 @@ def _required_runtime_env() -> dict[str, str]: "ROUTE_IMPL_HEALTH": "fastapi_native", "ROUTE_IMPL_SPECIFICATION": "fastapi_native", "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", **_v2_target_env(), **_gateway_auth_env(), } @@ -116,7 +133,19 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: "ROUTE_IMPL_HEALTH": "fastapi_native", "ROUTE_IMPL_SPECIFICATION": "fastapi_native", "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + PRODUCTION_CLOUD_SQL_INSTANCE + ), + "V2_SUPABASE_PROJECT_REF": TEST_V2_PROJECT_REF, + "V2_SUPABASE_ENVIRONMENT": TEST_V2_ENVIRONMENT, + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": ( + TEST_V2_RUNTIME_SECRET_RESOURCE + ), }, + "candidate_cloud_sql": PRODUCTION_CLOUD_SQL_INSTANCE, "updates": [], } ), @@ -155,7 +184,12 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: json.dumps( { "metadata": { - "labels": {"serving.knative.dev/service": revision_service} + "labels": {"serving.knative.dev/service": revision_service}, + "annotations": { + "run.googleapis.com/cloudsql-instances": state.get( + "candidate_cloud_sql", "" + ) + }, }, "spec": { "containers": [ @@ -505,12 +539,12 @@ def test_production_gunicorn_workers_do_not_inherit_database_pools(): def test_validate_cloud_run_deploy_env_requires_selector_environment_variable(): + env = _script_env(**_required_runtime_env()) + env.pop("SIM_ENTRYPOINT") + env.pop("SIMULATION_ENTRYPOINT_URL") result = _run_script( ".github/scripts/validate_cloud_run_deploy_env.sh", - _script_env( - OLD_SIMULATION_GATEWAY_URL="https://old-gateway.example.test", - **_gateway_auth_env(), - ), + env, ) assert result.returncode == 1 @@ -526,7 +560,24 @@ def test_validate_cloud_run_deploy_env_accepts_direct_mode_from_environment(): ROUTE_IMPL_HEALTH="fastapi_native", ROUTE_IMPL_SPECIFICATION="fastapi_native", ROUTE_IMPL_METADATA="fastapi_native", + ROUTE_IMPL_POLICY="flask_fallback", + DB_READ_POLICY="cloud_sql", + DB_WRITE_POLICY="cloud_sql", POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, + DEPLOYMENT_ENVIRONMENT="production", + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=( + PRODUCTION_CLOUD_SQL_INSTANCE + ), + PRODUCTION_V2_SUPABASE_PROJECT_REF=TEST_V2_PROJECT_REF, + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE=( + TEST_V2_RUNTIME_SECRET_RESOURCE + ), + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), **_v2_target_env(), **_gateway_auth_env(), ), @@ -541,9 +592,12 @@ def test_validate_cloud_run_deploy_env_accepts_direct_mode_from_environment(): "ROUTE_IMPL_HEALTH", "ROUTE_IMPL_SPECIFICATION", "ROUTE_IMPL_METADATA", + "ROUTE_IMPL_POLICY", + "DB_READ_POLICY", + "DB_WRITE_POLICY", ], ) -def test_validate_cloud_run_deploy_env_requires_stage6_selectors(missing_selector): +def test_validate_cloud_run_deploy_env_requires_migration_selectors(missing_selector): env = _script_env(**_required_runtime_env()) env.pop(missing_selector) @@ -562,9 +616,10 @@ def test_validate_cloud_run_deploy_env_requires_stage6_selectors(missing_selecto "ROUTE_IMPL_HEALTH", "ROUTE_IMPL_SPECIFICATION", "ROUTE_IMPL_METADATA", + "ROUTE_IMPL_POLICY", ], ) -def test_validate_cloud_run_deploy_env_rejects_invalid_stage6_selectors( +def test_validate_cloud_run_deploy_env_rejects_invalid_route_selectors( invalid_selector, ): env = _script_env(**_required_runtime_env()) @@ -582,6 +637,33 @@ def test_validate_cloud_run_deploy_env_rejects_invalid_stage6_selectors( ) in result.stderr +@pytest.mark.parametrize( + ("selector", "invalid_value", "expected_values"), + [ + ("DB_READ_POLICY", "supabase", "cloud_sql"), + ("DB_WRITE_POLICY", "supabase", "cloud_sql or dual_write"), + ], +) +def test_validate_cloud_run_deploy_env_rejects_invalid_policy_database_selectors( + selector, + invalid_value, + expected_values, +): + env = _script_env(**_required_runtime_env()) + env[selector] = invalid_value + + result = _run_script( + ".github/scripts/validate_cloud_run_deploy_env.sh", + env, + ) + + assert result.returncode == 1 + assert ( + f"{selector}={invalid_value} is invalid; expected {expected_values}" + in result.stderr + ) + + @pytest.mark.parametrize( ("entrypoint", "selected_url_env", "selected_url"), [ @@ -607,7 +689,24 @@ def test_validate_cloud_run_deploy_env_requires_only_selected_url( ROUTE_IMPL_HEALTH="fastapi_native", ROUTE_IMPL_SPECIFICATION="fastapi_native", ROUTE_IMPL_METADATA="fastapi_native", + ROUTE_IMPL_POLICY="flask_fallback", + DB_READ_POLICY="cloud_sql", + DB_WRITE_POLICY="cloud_sql", POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, + DEPLOYMENT_ENVIRONMENT="production", + PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=( + PRODUCTION_CLOUD_SQL_INSTANCE + ), + PRODUCTION_V2_SUPABASE_PROJECT_REF=TEST_V2_PROJECT_REF, + PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE=( + TEST_V2_RUNTIME_SECRET_RESOURCE + ), + PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), **_v2_target_env(), **_gateway_auth_env(), ) @@ -689,6 +788,129 @@ def test_deployment_validation_requires_database_instance_connection_name(): assert "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME" in result.stderr +def _staging_runtime_env() -> dict[str, str]: + return { + **_required_runtime_env(), + "DEPLOYMENT_ENVIRONMENT": "staging", + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": ( + "policyengine-api:us-central1:policyengine-api-data-staging" + ), + "CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET": ( + "policyengine-api-staging-db-password:latest" + ), + "V2_SUPABASE_PROJECT_REF": "z" * 20, + "V2_SUPABASE_ENVIRONMENT": "staging", + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": ( + "projects/test-project/secrets/v2-staging-runtime-url/versions/latest" + ), + } + + +def _phase10_staging_exercise_env() -> dict[str, str]: + return { + **_staging_runtime_env(), + "ROUTE_IMPL_POLICY": "fastapi_native", + "POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "policyengine-api-staging-db-readonly-password" + ), + "POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "policyengine-api-staging-db-migration-password" + ), + "PRODUCTION_POLICYENGINE_DB_READONLY_PASSWORD_SECRET": ( + "policyengine-api-prod-db-readonly-password" + ), + "PRODUCTION_POLICYENGINE_DB_MIGRATION_PASSWORD_SECRET": ( + "policyengine-api-prod-db-migration-password" + ), + "V2_FAILURE_DATABASE_URL_SECRET_RESOURCE": ( + "projects/test-project/secrets/v2-staging-unavailable-url/versions/latest" + ), + } + + +def test_deployment_validation_accepts_distinct_staging_database_targets(): + result = _run_script( + ".github/scripts/validate_cloud_run_deploy_env.sh", + _script_env(**_staging_runtime_env()), + ) + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("setting", "production_setting"), + [ + ( + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME", + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME", + ), + ("V2_SUPABASE_PROJECT_REF", "PRODUCTION_V2_SUPABASE_PROJECT_REF"), + ( + "CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET", + "PRODUCTION_CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET", + ), + ( + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", + "PRODUCTION_V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", + ), + ], +) +def test_deployment_validation_rejects_shared_staging_database_targets( + setting, + production_setting, +): + env = _staging_runtime_env() + env[setting] = env[production_setting] + + result = _run_script( + ".github/scripts/validate_cloud_run_deploy_env.sh", + _script_env(**env), + ) + + assert result.returncode == 1 + assert "distinct from production" in result.stderr + + +def test_phase10_staging_exercise_requires_isolated_activation_configuration(): + result = _run_script( + ".github/scripts/validate_phase10_staging_exercise_env.sh", + _script_env(**_phase10_staging_exercise_env()), + ) + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("setting", "invalid_value", "message"), + [ + ("DEPLOYMENT_ENVIRONMENT", "production", "only against staging"), + ("ROUTE_IMPL_POLICY", "flask_fallback", "must be fastapi_native"), + ("DB_READ_POLICY", "supabase", "must remain cloud_sql"), + ("DB_WRITE_POLICY", "dual_write", "must begin with"), + ( + "V2_FAILURE_DATABASE_URL_SECRET_RESOURCE", + "projects/test-project/secrets/v2-staging-runtime-url/versions/latest", + "must differ from the valid staging secret", + ), + ], +) +def test_phase10_staging_exercise_rejects_unsafe_configuration( + setting, + invalid_value, + message, +): + env = _phase10_staging_exercise_env() + env[setting] = invalid_value + + result = _run_script( + ".github/scripts/validate_phase10_staging_exercise_env.sh", + _script_env(**env), + ) + + assert result.returncode == 1 + assert message in result.stderr + + def test_build_cloud_run_image_dry_run_uses_cloud_run_dockerfile(): dockerignore = REPO / "gcp/cloud_run/Dockerfile.dockerignore" @@ -731,7 +953,7 @@ def test_deploy_cloud_run_candidate_dry_run_never_shifts_traffic(): f"--service-account {DEDICATED_CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" in result.stdout ) - assert f"--add-cloudsql-instances {PRODUCTION_CLOUD_SQL_INSTANCE}" in result.stdout + assert f"--set-cloudsql-instances {PRODUCTION_CLOUD_SQL_INSTANCE}" in result.stdout assert ( f"POLICYENGINE_DB_INSTANCE_CONNECTION_NAME={PRODUCTION_CLOUD_SQL_INSTANCE}" in result.stdout @@ -776,6 +998,24 @@ def test_deploy_cloud_run_candidate_dry_run_never_shifts_traffic(): "ROUTE_IMPL_METADATA", ): assert result.stdout.count(f"{selector}=fastapi_native") == 1 + assert result.stdout.count("ROUTE_IMPL_POLICY=flask_fallback") == 1 + assert result.stdout.count("DB_READ_POLICY=cloud_sql") == 1 + assert result.stdout.count("DB_WRITE_POLICY=cloud_sql") == 1 + + +def test_deploy_cloud_run_candidate_passes_optional_startup_warmup_setting(): + result = _run_script( + ".github/scripts/deploy_cloud_run_candidate.sh", + _script_env( + **_required_runtime_env(), + CLOUD_RUN_IMAGE_URI="us-central1-docker.pkg.dev/project/repo/api:sha", + CLOUD_RUN_TAG="stage3-test", + POLICYENGINE_API_STARTUP_WARMUP="0", + ), + ) + + assert result.returncode == 0, result.stderr + assert "POLICYENGINE_API_STARTUP_WARMUP=0" in result.stdout def test_staging_and_production_use_distinct_cloud_run_runtime_identities(): @@ -796,6 +1036,7 @@ def test_deploy_cloud_run_candidate_uses_configured_database_instance(): env = { **_required_runtime_env(), "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": configured_instance, + "PRODUCTION_POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": configured_instance, } result = _run_script( ".github/scripts/deploy_cloud_run_candidate.sh", @@ -807,7 +1048,7 @@ def test_deploy_cloud_run_candidate_uses_configured_database_instance(): ) assert result.returncode == 0, result.stderr - assert f"--add-cloudsql-instances {configured_instance}" in result.stdout + assert f"--set-cloudsql-instances {configured_instance}" in result.stdout assert ( f"POLICYENGINE_DB_INSTANCE_CONNECTION_NAME={configured_instance}" in result.stdout @@ -952,13 +1193,16 @@ def test_resolve_cloud_run_candidate_rejects_changed_image(tmp_path): assert "Candidate image changed" in result.stderr -def test_resolve_cloud_run_candidate_verifies_stage6_route_selectors(tmp_path): +def test_resolve_cloud_run_candidate_verifies_deployment_selectors(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) env = { **_fake_gcloud_env(gcloud_path, state_path), "ROUTE_IMPL_HEALTH": "fastapi_native", "ROUTE_IMPL_SPECIFICATION": "fastapi_native", "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", } result = _run_script( @@ -969,7 +1213,7 @@ def test_resolve_cloud_run_candidate_verifies_stage6_route_selectors(tmp_path): assert result.returncode == 0, result.stderr -def test_resolve_cloud_run_candidate_rejects_stage6_selector_mismatch(tmp_path): +def test_resolve_cloud_run_candidate_rejects_deployment_selector_mismatch(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) state = json.loads(state_path.read_text(encoding="utf-8")) state["candidate_env"]["ROUTE_IMPL_METADATA"] = "flask_fallback" @@ -982,6 +1226,9 @@ def test_resolve_cloud_run_candidate_rejects_stage6_selector_mismatch(tmp_path): "ROUTE_IMPL_HEALTH": "fastapi_native", "ROUTE_IMPL_SPECIFICATION": "fastapi_native", "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", }, ) @@ -992,6 +1239,87 @@ def test_resolve_cloud_run_candidate_rejects_stage6_selector_mismatch(tmp_path): ) in result.stderr +def test_resolve_cloud_run_candidate_rejects_policy_write_selector_mismatch( + tmp_path, +): + gcloud_path, state_path = _fake_gcloud(tmp_path) + state = json.loads(state_path.read_text(encoding="utf-8")) + state["candidate_env"]["DB_WRITE_POLICY"] = "dual_write" + state_path.write_text(json.dumps(state), encoding="utf-8") + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + { + **_fake_gcloud_env(gcloud_path, state_path), + "ROUTE_IMPL_HEALTH": "fastapi_native", + "ROUTE_IMPL_SPECIFICATION": "fastapi_native", + "ROUTE_IMPL_METADATA": "fastapi_native", + "ROUTE_IMPL_POLICY": "flask_fallback", + "DB_READ_POLICY": "cloud_sql", + "DB_WRITE_POLICY": "cloud_sql", + }, + ) + + assert result.returncode == 2 + assert ( + "Revision policyengine-api-00002-new has DB_WRITE_POLICY=dual_write; " + "expected cloud_sql" + ) in result.stderr + + +def test_resolve_cloud_run_candidate_verifies_database_identities(tmp_path): + gcloud_path, state_path = _fake_gcloud(tmp_path) + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + { + **_fake_gcloud_env(gcloud_path, state_path), + **_v2_target_env(), + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": (PRODUCTION_CLOUD_SQL_INSTANCE), + }, + ) + + assert result.returncode == 0, result.stderr + + +def test_resolve_cloud_run_candidate_rejects_database_identity_mismatch(tmp_path): + gcloud_path, state_path = _fake_gcloud(tmp_path) + state = json.loads(state_path.read_text(encoding="utf-8")) + state["candidate_env"]["V2_SUPABASE_PROJECT_REF"] = "z" * 20 + state_path.write_text(json.dumps(state), encoding="utf-8") + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + { + **_fake_gcloud_env(gcloud_path, state_path), + **_v2_target_env(), + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": (PRODUCTION_CLOUD_SQL_INSTANCE), + }, + ) + + assert result.returncode == 2 + assert "V2_SUPABASE_PROJECT_REF=zzzz" in result.stderr + + +def test_resolve_cloud_run_candidate_rejects_cloud_sql_attachment_mismatch(tmp_path): + gcloud_path, state_path = _fake_gcloud(tmp_path) + state = json.loads(state_path.read_text(encoding="utf-8")) + state["candidate_cloud_sql"] = "project:region:wrong-instance" + state_path.write_text(json.dumps(state), encoding="utf-8") + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + { + **_fake_gcloud_env(gcloud_path, state_path), + **_v2_target_env(), + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": (PRODUCTION_CLOUD_SQL_INSTANCE), + }, + ) + + assert result.returncode == 2 + assert "attaches Cloud SQL instance project:region:wrong-instance" in result.stderr + + def test_set_cloud_run_revision_promotes_and_rolls_back_exact_revisions(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) env = _fake_gcloud_env(gcloud_path, state_path) @@ -1324,6 +1652,7 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): "integration-tests-staging-cloud-run", ) cloud_run_promotion = _workflow_job_block(workflow, "promote-cloud-run-staging") + phase10_exercise = _workflow_job_block(workflow, "exercise-phase10-staging") production_gate = _workflow_job_block( workflow, "ensure-production-model-version-aligns-with-sim-api", @@ -1331,6 +1660,7 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): cloud_run_test_command = ( "python -m pytest tests/integration/test_cloud_run_candidate.py " "tests/integration/test_live_v2_metadata.py " + "tests/integration/test_live_v2_policies.py " "tests/integration/test_live_calculate.py " "tests/integration/test_live_economy.py " "tests/integration/test_live_budget_window_cache.py -v" @@ -1348,9 +1678,21 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): "API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}" in cloud_run_tests ) - assert "needs: promote-cloud-run-staging" in production_gate + assert "environment: staging" in cloud_run_tests + assert "V2_MIGRATION_DATABASE_URL" in cloud_run_tests + assert "needs: exercise-phase10-staging" in production_gate assert "- integration-tests-staging-cloud-run" not in production_gate assert "- integration-tests-staging-cloud-run" in cloud_run_promotion + assert "- promote-cloud-run-staging" in phase10_exercise + assert "DB_WRITE_POLICY: dual_write" in phase10_exercise + assert "Deploy controlled-failure staging revision" in phase10_exercise + assert 'POLICYENGINE_API_STARTUP_WARMUP: "0"' in phase10_exercise + assert "V2_FAILURE_DATABASE_URL_SECRET_RESOURCE" in phase10_exercise + assert "run_phase10_staging_probe.sh activation" in phase10_exercise + assert "run_phase10_staging_probe.sh rollback" in phase10_exercise + assert "Restore exact Cloud SQL-only staging revision" in phase10_exercise + assert "actions/upload-artifact@v4" in phase10_exercise + assert "Fail after an incomplete Phase 10 staging exercise" in phase10_exercise assert "qualify-stage6-read-routes-staging" not in workflow assert "qualify_stage6_read_routes.sh" not in workflow assert "bash .github/scripts/set_cloud_run_revision.sh" in cloud_run_promotion @@ -1389,6 +1731,9 @@ def test_push_workflow_uses_local_redis_for_predeployment_test_suite(): assert "RUNTIME_CACHE_ENVIRONMENT: test" in test_step assert "RUNTIME_CACHE_SERVICE: api" in test_step assert "-u ROUTE_IMPL_HEALTH" in test_step + assert "-u ROUTE_IMPL_POLICY" in test_step + assert "-u DB_READ_POLICY" in test_step + assert "-u DB_WRITE_POLICY" in test_step assert "-u CLOUD_RUN_SERVICE" in test_step assert "-u V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE" in test_step @@ -1465,16 +1810,20 @@ def test_workflows_scope_simulation_routing_config_to_github_environments(): assert secret_env in job -def test_cloud_run_deploy_jobs_use_environment_scoped_stage6_route_selectors(): +def test_cloud_run_candidate_jobs_use_environment_scoped_migration_selectors(): workflow = _push_workflow() selectors = ( "ROUTE_IMPL_HEALTH", "ROUTE_IMPL_SPECIFICATION", "ROUTE_IMPL_METADATA", + "ROUTE_IMPL_POLICY", + "DB_READ_POLICY", + "DB_WRITE_POLICY", ) for job_name, environment in ( ("deploy-cloud-run-staging", "staging"), + ("promote-cloud-run-staging", "staging"), ("deploy-cloud-run-candidate", "production"), ): job = _workflow_job_block(workflow, job_name) diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index 9517db7bc..431107bdd 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -10,13 +10,16 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["version"] == 1 assert payload["metadata"] == { "route_group_count": 9, - "workflow_count": 8, - "request_count": 32, + "workflow_count": 11, + "request_count": 43, "db_entity_count": 6, "sim_flow_count": 3, } assert {workflow["name"] for workflow in payload["workflows"]} == { "policy_save_search", + "policy_resources_v2", + "saved_policy_v1_compatibility", + "user_policy_associations_v2", "household_save_edit_read", "household_calculate", "region_selection", diff --git a/tests/unit/test_migration_flags.py b/tests/unit/test_migration_flags.py index c70ec95ff..09ddc9bd2 100644 --- a/tests/unit/test_migration_flags.py +++ b/tests/unit/test_migration_flags.py @@ -104,6 +104,25 @@ def test_invalid_migration_flag_raises(monkeypatch): get_migration_context("policy") +def test_phase10_v1_policy_sources_allow_only_cloud_sql_reads_and_mirroring( + monkeypatch, +): + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + + assert migration_flags.get_v1_policy_write_source() == "dual_write" + assert migration_flags.get_v1_policy_read_source() == "cloud_sql" + + monkeypatch.setenv("DB_WRITE_POLICY", "supabase") + with pytest.raises(ValueError, match="DB_WRITE_POLICY"): + migration_flags.get_v1_policy_write_source() + + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + monkeypatch.setenv("DB_READ_POLICY", "read_compare") + with pytest.raises(ValueError, match="DB_READ_POLICY"): + migration_flags.get_v1_policy_read_source() + + @pytest.mark.parametrize( "explicit_sources", [ @@ -134,6 +153,13 @@ def test_explicit_migration_context_rejects_invalid_database_sources( ("/v2/variables", "metadata"), ("/v2/parameters/children", "metadata"), ("/v2/regions/state%2Fca", "metadata"), + ("/v2/policies", "policy"), + ("/v2/policies/00000000-0000-0000-0000-000000000001", "policy"), + ("/v2/user-policies", "policy"), + ( + "/v2/user-policies/00000000-0000-0000-0000-000000000001", + "policy", + ), ("/us/policy/1", "policy"), ("/us/policies", "policy"), ("/us/household/1", "household"), diff --git a/tests/unit/test_query_parameters.py b/tests/unit/test_query_parameters.py new file mode 100644 index 000000000..fecc2748b --- /dev/null +++ b/tests/unit/test_query_parameters.py @@ -0,0 +1,222 @@ +"""Shared query-schema and framework-adapter tests.""" + +from __future__ import annotations + +from typing import Annotated +from uuid import UUID, uuid4 + +from fastapi import Depends, FastAPI +from fastapi.testclient import TestClient +from pydantic import Field, ValidationError +import pytest +from werkzeug.datastructures import MultiDict + +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + CountryQuery, + DuplicateScalarQueryParameterError, + PolicyCollectionQuery, + PolicyCreateQuery, + ResourceId, + UserPolicyCollectionQuery, + parse_multidict_query, + parse_query_items, +) + + +class ExplicitListQuery(CountryQuery): + policy_ids: Annotated[ + list[ResourceId], + Field(default_factory=list, max_length=10), + ] + + +def test_canonical_required_defaults_and_normalization() -> None: + parsed = parse_query_items(PolicyCollectionQuery, [("country_id", "US")]) + + assert parsed.country_id == "us" + assert parsed.offset == 0 + assert parsed.limit == 100 + assert parsed.tax_benefit_model_id is None + + with pytest.raises(ValidationError) as missing: + parse_query_items(PolicyCollectionQuery, []) + assert missing.value.errors()[0]["loc"] == ("country_id",) + + +@pytest.mark.parametrize( + ("items", "field"), + [ + ([("country_id", "ca")], "country_id"), + ([("country_id", "us"), ("offset", "-1")], "offset"), + ([("country_id", "us"), ("limit", "0")], "limit"), + ([("country_id", "us"), ("limit", "501")], "limit"), + ( + [("country_id", "us"), ("tax_benefit_model_id", "not-a-uuid")], + "tax_benefit_model_id", + ), + ([("country_id", "us"), ("unexpected", "value")], "unexpected"), + ], +) +def test_canonical_malformed_out_of_range_and_unknown_values( + items: list[tuple[str, str]], + field: str, +) -> None: + with pytest.raises(ValidationError) as raised: + parse_query_items(PolicyCollectionQuery, items) + assert raised.value.errors()[0]["loc"] == (field,) + + +@pytest.mark.parametrize("version", ["", " 5.2.0", "v5.2.0", "0.0.0"]) +def test_policyengine_version_must_be_canonical(version: str) -> None: + with pytest.raises(ValidationError): + parse_query_items( + PolicyCreateQuery, + [("country_id", "uk"), ("policyengine_version", version)], + ) + + assert ( + parse_query_items( + PolicyCreateQuery, + [("country_id", "uk"), ("policyengine_version", "5.2.0")], + ).policyengine_version + == "5.2.0" + ) + + +def test_duplicate_scalar_is_rejected_and_explicit_list_is_preserved() -> None: + with pytest.raises( + DuplicateScalarQueryParameterError, + match="country_id.*must not be repeated", + ): + parse_query_items( + PolicyCollectionQuery, + [("country_id", "us"), ("country_id", "uk")], + ) + + first_id = uuid4() + second_id = uuid4() + parsed = parse_query_items( + ExplicitListQuery, + [ + ("country_id", "us"), + ("policy_ids", str(first_id)), + ("policy_ids", str(second_id)), + ], + ) + assert parsed.policy_ids == [first_id, second_id] + + +def test_multidict_adapter_preserves_the_canonical_contract() -> None: + policy_id = uuid4() + user_id = uuid4() + parsed = parse_multidict_query( + UserPolicyCollectionQuery, + MultiDict( + [ + ("country_id", "UK"), + ("user_id", str(user_id)), + ("policy_id", str(policy_id)), + ("offset", "2"), + ] + ), + ) + + assert parsed.country_id == "uk" + assert parsed.user_id == user_id + assert parsed.policy_id == policy_id + assert parsed.offset == 2 + + +def _test_client() -> TestClient: + app = FastAPI() + dependency = query_dependency(ExplicitListQuery) + + @app.get("/resources") + def resources( + query: ExplicitListQuery = Depends(dependency), + ) -> dict[str, object]: + return query.model_dump(mode="json") + + return TestClient(app) + + +def test_fastapi_dependency_rejects_unknown_and_duplicate_scalar_parameters() -> None: + client = _test_client() + + assert client.get("/resources?country_id=us&unknown=value").status_code == 422 + assert client.get("/resources?country_id=us&country_id=uk").status_code == 422 + + +def test_fastapi_dependency_accepts_repeated_explicit_list_parameters() -> None: + first_id = uuid4() + second_id = uuid4() + + response = _test_client().get( + f"/resources?country_id=US&policy_ids={first_id}&policy_ids={second_id}" + ) + + assert response.status_code == 200 + assert response.json() == { + "country_id": "us", + "policy_ids": [str(first_id), str(second_id)], + } + + +def test_openapi_matches_composed_runtime_query_contract() -> None: + operation = _test_client().get("/openapi.json").json()["paths"]["/resources"]["get"] + parameters = {item["name"]: item for item in operation["parameters"]} + + assert set(parameters) == {"country_id", "policy_ids"} + assert parameters["country_id"]["required"] is True + assert parameters["country_id"]["schema"]["enum"] == ["us", "uk"] + assert parameters["policy_ids"]["required"] is False + assert parameters["policy_ids"]["schema"]["type"] == "array" + assert parameters["policy_ids"]["schema"]["maxItems"] == 10 + assert parameters["policy_ids"]["schema"]["items"]["format"] == "uuid" + + +def test_policy_collection_openapi_defaults_and_bounds() -> None: + app = FastAPI() + dependency = query_dependency(PolicyCollectionQuery) + + @app.get("/policies") + def policies( + query: PolicyCollectionQuery = Depends(dependency), + ) -> dict[str, object]: + return query.model_dump(mode="json") + + operation = TestClient(app).get("/openapi.json").json()["paths"]["/policies"]["get"] + parameters = {item["name"]: item for item in operation["parameters"]} + + assert parameters["offset"]["schema"] == { + "type": "integer", + "minimum": 0, + "description": "Zero-based result offset", + "default": 0, + "title": "Offset", + } + assert parameters["limit"]["schema"] == { + "type": "integer", + "maximum": 500, + "minimum": 1, + "description": "Maximum number of returned resources", + "default": 100, + "title": "Limit", + } + assert parameters["tax_benefit_model_id"]["required"] is False + uuid_schema = parameters["tax_benefit_model_id"]["schema"]["anyOf"][0] + assert uuid_schema == { + "type": "string", + "format": "uuid", + "description": "Exact resource UUID", + } + + +def test_uuid_filter_is_materialized_as_uuid() -> None: + model_id = uuid4() + parsed = parse_query_items( + PolicyCollectionQuery, + [("country_id", "us"), ("tax_benefit_model_id", str(model_id))], + ) + assert isinstance(parsed.tax_benefit_model_id, UUID) diff --git a/tests/unit/test_readiness.py b/tests/unit/test_readiness.py index 1f5f706c6..3cc29168c 100644 --- a/tests/unit/test_readiness.py +++ b/tests/unit/test_readiness.py @@ -1,4 +1,5 @@ import pytest +from unittest.mock import patch from policyengine_api import readiness @@ -19,3 +20,48 @@ def test_mark_not_ready_then_ready(): assert readiness.is_ready() is False readiness.mark_ready() assert readiness.is_ready() is True + + +def test_default_cloud_sql_policy_mode_does_not_require_v2_settings( + monkeypatch, +): + monkeypatch.delenv("DB_WRITE_POLICY", raising=False) + monkeypatch.delenv("DB_READ_POLICY", raising=False) + monkeypatch.delenv("ROUTE_IMPL_POLICY", raising=False) + monkeypatch.delenv("V2_RUNTIME_DATABASE_URL", raising=False) + monkeypatch.delenv("V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", raising=False) + + readiness.mark_ready() + + assert readiness.is_ready() is True + + +def test_dual_write_or_native_policy_routes_require_v2_runtime_settings( + monkeypatch, +): + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + monkeypatch.delenv("V2_RUNTIME_DATABASE_URL", raising=False) + monkeypatch.delenv("V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", raising=False) + readiness.mark_ready() + + assert readiness.is_ready() is False + + +def test_selected_v2_policy_modes_validate_runtime_settings(monkeypatch): + monkeypatch.setenv("DB_WRITE_POLICY", "dual_write") + monkeypatch.setenv("DB_READ_POLICY", "cloud_sql") + monkeypatch.setenv("ROUTE_IMPL_POLICY", "flask_fallback") + readiness.mark_ready() + + with patch( + "policyengine_api.data.v2.settings.load_v2_runtime_database_settings", + return_value=object(), + ) as load_settings: + assert readiness.is_ready() is True + + load_settings.assert_called_once_with() + + monkeypatch.setenv("DB_WRITE_POLICY", "cloud_sql") + monkeypatch.setenv("ROUTE_IMPL_POLICY", "fastapi_native") + + assert readiness.is_ready() is False diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 815acf476..934fc1614 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -217,11 +217,14 @@ def test_v2_files_are_mechanically_separate_from_v1() -> None: assert all("migrations/v1" not in str(path) for path in v2_files) -def test_v2_revision_chain_has_baseline_and_generated_stage_9_revision() -> None: +def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> None: config = Config(str(REPO / "alembic-v2.ini")) script = ScriptDirectory.from_config(config) - assert script.get_heads() == ["68b4a5ae5dc5"] + assert script.get_heads() == ["af34023a728f"] assert [revision.revision for revision in script.walk_revisions()] == [ + "af34023a728f", + "c21c4a807a49", + "711ec2f0a5a5", "68b4a5ae5dc5", "f5ef4347cb2a", ] @@ -248,8 +251,8 @@ def test_v2_revision_chain_has_baseline_and_generated_stage_9_revision() -> None assert "fk_regions_default_dataset_model_datasets" in baseline assert "uq_datasets_model_name" in baseline assert "ck_datasets_output_storage_path" in baseline - assert baseline.count("op.create_table(") == len(V2_TABLE_NAMES) - assert baseline.count("op.drop_table(") == len(V2_TABLE_NAMES) + assert baseline.count("op.create_table(") == len(V2_TABLE_NAMES) - 3 + assert baseline.count("op.drop_table(") == len(V2_TABLE_NAMES) - 3 corrected_enum_names = set( re.findall( @@ -288,6 +291,80 @@ def test_v2_revision_chain_has_baseline_and_generated_stage_9_revision() -> None assert "op.bulk_insert(" not in stage_9_revision +def test_phase_10_revision_has_only_documented_generation_corrections() -> None: + revision = ( + REPO / "migrations/v2/versions/711ec2f0a5a5_migrate_v2_policies.py" + ).read_text(encoding="utf-8") + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert 'down_revision: Union[str, None] = "68b4a5ae5dc5"' in revision + assert revision.count("Post-generation correction:") == 4 + assert 'postgresql_using="user_id::text"' in revision + assert 'postgresql_using="user_id::uuid"' in revision + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision + + policy_key = revision.index('"uq_policies_id_country"') + policy_mapping = revision.index( + 'op.create_table(\n "legacy_policy_mappings"' + ) + association_key = revision.index('"uq_user_policies_id_country"') + association_mapping = revision.index( + 'op.create_table(\n "legacy_user_policy_mappings"' + ) + drop_association_mapping = revision.index( + 'op.drop_table("legacy_user_policy_mappings")' + ) + drop_association_key = revision.index( + 'op.drop_constraint("uq_user_policies_id_country"' + ) + + assert policy_key < policy_mapping + assert association_key < association_mapping + assert drop_association_mapping < drop_association_key + + +def test_saved_policy_revision_tracking_was_generated_after_phase_10() -> None: + revision = ( + REPO + / "migrations/v2/versions/c21c4a807a49_track_saved_policy_mirror_revisions.py" + ).read_text(encoding="utf-8") + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert 'down_revision: Union[str, None] = "711ec2f0a5a5"' in revision + assert "last_applied_source_revision" in revision + assert "ck_legacy_user_policy_mappings_source_revision" in revision + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision + + +def test_legacy_user_uuid_mapping_revision_is_generated_and_reversible() -> None: + revision = ( + REPO / "migrations/v2/versions/af34023a728f_map_legacy_users_to_v2_uuids.py" + ).read_text(encoding="utf-8") + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert 'down_revision: Union[str, None] = "c21c4a807a49"' in revision + assert revision.count("Post-generation correction:") == 2 + assert 'postgresql_using="user_id::uuid"' in revision + assert 'postgresql_using="user_id::text"' in revision + assert 'op.create_table(\n "legacy_user_mappings"' in revision + assert 'op.drop_table("legacy_user_mappings")' in revision + assert "fk_user_policies_user_id_users" in revision + assert "uq_legacy_user_mappings_user_id" in revision + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision + + def test_alembic_rejects_unknown_missing_and_divergent_history(tmp_path: Path) -> None: original = REPO / "migrations/v2" missing = tmp_path / "missing" diff --git a/tests/unit/v2/test_data_crud_boundaries.py b/tests/unit/v2/test_data_crud_boundaries.py new file mode 100644 index 000000000..fcef5b1b1 --- /dev/null +++ b/tests/unit/v2/test_data_crud_boundaries.py @@ -0,0 +1,110 @@ +"""Structural checks for API v2 service and database-connector modules.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + + +PROJECT_ROOT = Path(__file__).parents[3] +SERVICES_ROOT = PROJECT_ROOT / "policyengine_api" / "services" / "v2" + + +def _tree(path: Path) -> ast.Module: + return ast.parse(path.read_text(encoding="utf-8")) + + +def _called_names(path: Path) -> set[str]: + names: set[str] = set() + for node in ast.walk(_tree(path)): + if not isinstance(node, ast.Call): + continue + if isinstance(node.func, ast.Name): + names.add(node.func.id) + elif isinstance(node.func, ast.Attribute): + names.add(node.func.attr) + return names + + +def _imported_modules(path: Path) -> set[str]: + modules: set[str] = set() + for node in ast.walk(_tree(path)): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + modules.add(node.module or "") + return modules + + +@pytest.mark.parametrize( + "relative_path", + ( + "policies/database_connectors/creates.py", + "user_policies/database_connectors/creates.py", + "user_policies/database_connectors/updates.py", + "user_policies/database_connectors/deletes.py", + ), +) +def test_mutation_connectors_contain_no_database_reads(relative_path: str) -> None: + calls = _called_names(SERVICES_ROOT / relative_path) + assert "select" not in calls + assert "get" not in calls + + +@pytest.mark.parametrize( + "relative_path", + ( + "policies/database_connectors/reads.py", + "user_policies/database_connectors/reads.py", + "metadata/database_connectors/reads.py", + "metadata/database_connectors/reads_datasets.py", + "metadata/database_connectors/reads_parameter_tree.py", + "metadata/database_connectors/reads_parameters.py", + "metadata/database_connectors/reads_regions.py", + "metadata/database_connectors/reads_variables.py", + ), +) +def test_read_connectors_contain_no_database_mutations(relative_path: str) -> None: + calls = _called_names(SERVICES_ROOT / relative_path) + assert calls.isdisjoint({"insert", "add", "add_all", "delete"}) + + +@pytest.mark.parametrize( + "relative_path", + ( + "policies/validators.py", + "policies/transformations.py", + "metadata/validators.py", + "metadata/transformations.py", + "user_policies/validators.py", + "user_policies/transformations.py", + ), +) +def test_validation_and_transformation_modules_have_no_database_query_dependency( + relative_path: str, +) -> None: + imports = _imported_modules(SERVICES_ROOT / relative_path) + assert not any( + module == "sqlalchemy" + or module.startswith("sqlalchemy.") + or module == "sqlmodel" + or module.startswith("sqlmodel.") + for module in imports + ) + + +@pytest.mark.parametrize( + "relative_path", + ( + "policies/database_session.py", + "metadata/database_session.py", + "user_policies/database_session.py", + ), +) +def test_database_session_modules_do_not_construct_queries( + relative_path: str, +) -> None: + calls = _called_names(SERVICES_ROOT / relative_path) + assert calls.isdisjoint({"select", "insert", "update", "delete", "exec"}) diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py index e9a153e38..bbc8179de 100644 --- a/tests/unit/v2/test_database.py +++ b/tests/unit/v2/test_database.py @@ -50,6 +50,34 @@ def test_engine_construction_is_lazy_and_reused_without_connecting() -> None: assert first.pool.checkedout() == 0 +def test_engine_uses_one_timeout_for_pool_connection_and_statements( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = load_v2_runtime_database_settings(_environment()) + captured: dict[str, object] = {} + expected_engine = object() + + def capture_create_engine(url: object, **options: object) -> object: + captured["url"] = url + captured.update(options) + return expected_engine + + monkeypatch.setattr(database, "create_engine", capture_create_engine) + + engine = database.build_v2_engine(settings) + + assert engine is expected_engine + assert captured["url"] is settings.connection.url + assert database.DATABASE_TIMEOUT_SECONDS == 5 + assert captured["pool_timeout"] == database.DATABASE_TIMEOUT_SECONDS + assert captured["connect_args"] == { + "connect_timeout": database.DATABASE_TIMEOUT_SECONDS, + "options": ( + f"-c statement_timeout={database.DATABASE_TIMEOUT_SECONDS * 1_000}" + ), + } + + def test_session_factory_builds_sqlmodel_sessions() -> None: settings = load_v2_runtime_database_settings(_environment()) factory = database.get_v2_session_factory(settings) diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index d64af2741..89f7923c3 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -50,11 +50,16 @@ def test_importing_v2_modules_opens_no_network_and_creates_no_files( script = """ import pathlib import socket +import sqlalchemy def reject_connect(*args, **kwargs): raise AssertionError("module import attempted a network connection") +def reject_ddl(*args, **kwargs): + raise AssertionError("module import attempted implicit DDL") + socket.socket.connect = reject_connect +sqlalchemy.MetaData.create_all = reject_ddl before = set(pathlib.Path.cwd().iterdir()) import policyengine_api.data.v2.settings import policyengine_api.data.v2.database @@ -62,8 +67,9 @@ def reject_connect(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 32 +assert len(V2_METADATA.tables) == 35 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules +assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules """ result = subprocess.run( diff --git a/tests/unit/v2/test_metadata_deployment.py b/tests/unit/v2/test_metadata_deployment.py index dedb61df9..03c5c025c 100644 --- a/tests/unit/v2/test_metadata_deployment.py +++ b/tests/unit/v2/test_metadata_deployment.py @@ -69,21 +69,29 @@ def test_schema_upgrade_precedes_atomic_catalog_publication() -> None: assert upgrade < current < drift +def test_policy_data_qualification_precedes_schema_upgrade() -> None: + workflow = _read(".github/workflows/seed-v2-database.yml") + + isolation = workflow.index("validate_database_environment.sh supabase") + qualification = workflow.index("scripts/qualify_v2_policy_migration.py") + upgrade = workflow.index("migrate_v2_metadata_schema.sh") + + assert isolation < qualification < upgrade + + def test_seeding_success_is_required_before_candidate_creation() -> None: workflow = _read(".github/workflows/push.yml") staging_seed = _job(workflow, "seed-v2-staging-database") production_seed = _job(workflow, "seed-v2-production-database") assert "deployment_environment: staging" in staging_seed - assert "migrate-v1-cloud-sql" in staging_seed + assert "migrate-v1-staging-cloud-sql" in staging_seed assert "seed-v2-staging-database" in _job( workflow, "deploy-cloud-run-staging", ) - assert ( - "needs: ensure-production-model-version-aligns-with-sim-api" in production_seed - ) + assert "migrate-v1-production-cloud-sql" in production_seed assert "deployment_environment: production" in production_seed assert "needs: seed-v2-production-database" in _job( workflow, diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 04c960969..afbf54d56 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -11,15 +11,13 @@ import pytest from policyengine_api.asgi_factory import create_asgi_app -from policyengine_api.data.v2.catalog.query import ( - InvalidMetadataPageError, +from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, - MetadataResourceNotFoundError, UnsupportedPreviewCountryError, ) -from policyengine_api.data.v2.catalog.schemas import ( +from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, MetadataDataset, MetadataDatasetOption, @@ -36,6 +34,10 @@ MetadataTimePeriodOption, MetadataVariable, ) +from policyengine_api.services.v2.metadata.validators import ( + InvalidMetadataPageError, + MetadataResourceNotFoundError, +) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes import dependencies as route_dependencies from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -370,7 +372,7 @@ def factory(): assert response.status_code == 422 assert response.json() == { "status": "error", - "message": "Invalid v2 metadata request", + "message": "Invalid API v2 request", } assert calls == [] @@ -435,19 +437,19 @@ def test_default_reader_uses_the_installed_policyengine_version( monkeypatch: pytest.MonkeyPatch, ) -> None: from policyengine_api.data.v2 import database - from policyengine_api.data.v2.catalog import query + from policyengine_api.services.v2.metadata import services as metadata_services session = object() captured = {} reader = object() def query_service(candidate_session, *, running_policyengine_version): - captured["session"] = candidate_session + captured["session"] = candidate_session.session captured["version"] = running_policyengine_version return reader monkeypatch.setattr(database, "get_v2_session_factory", lambda: lambda: session) - monkeypatch.setattr(query, "V2MetadataQueryService", query_service) + monkeypatch.setattr(metadata_services, "V2MetadataService", query_service) monkeypatch.setattr( route_dependencies.importlib_metadata, "version", @@ -515,7 +517,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: assert response.status_code == 200 schema = response.json() - expected_paths = { + metadata_paths = { "/v2/datasets", "/v2/datasets/{dataset_id}", "/v2/economy-options", @@ -535,9 +537,21 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/variables", "/v2/variables/{variable_id}", } - assert set(schema["paths"]) == expected_paths - - for path in expected_paths: + native_paths = { + "/v2/policies", + "/v2/policies/{policy_id}", + "/v2/user-policies", + "/v2/user-policies/{association_id}", + } + assert set(schema["paths"]) == metadata_paths | native_paths + assert "V2ErrorResponse" in schema["components"]["schemas"] + assert { + "MetadataErrorResponse", + "PolicyErrorResponse", + "UserPolicyErrorResponse", + }.isdisjoint(schema["components"]["schemas"]) + + for path in metadata_paths: operation = schema["paths"][path]["get"] assert set(operation["responses"]) >= { "200", @@ -556,4 +570,4 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: error_schema = operation["responses"][status]["content"][ "application/json" ]["schema"] - assert error_schema["$ref"] == "#/components/schemas/MetadataErrorResponse" + assert error_schema["$ref"] == "#/components/schemas/V2ErrorResponse" diff --git a/tests/unit/v2/test_metadata_query.py b/tests/unit/v2/test_metadata_service.py similarity index 90% rename from tests/unit/v2/test_metadata_query.py rename to tests/unit/v2/test_metadata_service.py index 2b57275eb..fe1069cb5 100644 --- a/tests/unit/v2/test_metadata_query.py +++ b/tests/unit/v2/test_metadata_service.py @@ -1,4 +1,4 @@ -"""Read-only query coverage for v2 metadata resources.""" +"""Read-only service coverage for v2 metadata resources.""" from __future__ import annotations @@ -12,14 +12,19 @@ from sqlalchemy.pool import StaticPool from sqlmodel import Session, create_engine, select -from policyengine_api.data.v2.catalog.query import ( - InvalidMetadataPageError, +from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, MetadataCatalogUnavailableError, MetadataCatalogVersionNotFoundError, - MetadataResourceNotFoundError, UnsupportedPreviewCountryError, - V2MetadataQueryService, +) +from policyengine_api.services.v2.metadata.database_session import ( + MetadataDatabaseSession, +) +from policyengine_api.services.v2.metadata.services import V2MetadataService +from policyengine_api.services.v2.metadata.validators import ( + InvalidMetadataPageError, + MetadataResourceNotFoundError, ) from policyengine_api.data.v2.models import ( Dataset, @@ -201,9 +206,9 @@ def _us_model_version(session: Session) -> TaxBenefitModelVersion: ).one() -def _service(session: Session) -> V2MetadataQueryService: - return V2MetadataQueryService( - session, +def _service(session: Session) -> V2MetadataService: + return V2MetadataService( + MetadataDatabaseSession(session), running_policyengine_version=POLICYENGINE_VERSION, ) @@ -485,8 +490,8 @@ def test_version_selection_rejects_invalid_absent_and_unsupported_requests( with pytest.raises(UnsupportedPreviewCountryError): service.list_variables("ca") with pytest.raises(MetadataCatalogUnavailableError): - V2MetadataQueryService( - catalog_session, + V2MetadataService( + MetadataDatabaseSession(catalog_session), running_policyengine_version="4.99.0", ).list_variables("us") @@ -551,24 +556,23 @@ def test_economy_options_require_a_national_region_and_dataset( _service(catalog_session).get_economy_options("us") -def test_query_modules_import_no_policyengine_or_v1_metadata_source() -> None: - source_directory = ( - Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" / "catalog" +def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: + project_package = Path(__file__).parents[3] / "policyengine_api" + connector_directory = ( + project_package / "services" / "v2" / "metadata" / "database_connectors" ) modules = ( - "catalog_selection.py", - "dataset_query.py", - "model_query.py", - "parameter_query.py", - "parameter_tree_query.py", - "query.py", - "query_support.py", - "region_query.py", - "variable_query.py", + project_package / "data" / "v2" / "catalog" / "catalog_selection.py", + connector_directory / "reads.py", + connector_directory / "reads_datasets.py", + connector_directory / "reads_parameter_tree.py", + connector_directory / "reads_parameters.py", + connector_directory / "reads_regions.py", + connector_directory / "reads_variables.py", ) imported = set() for module in modules: - tree = ast.parse((source_directory / module).read_text(encoding="utf-8")) + tree = ast.parse(module.read_text(encoding="utf-8")) imported.update( alias.name for node in ast.walk(tree) @@ -596,28 +600,28 @@ def test_query_modules_import_no_policyengine_or_v1_metadata_source() -> None: ) -def test_resource_service_methods_are_defined_in_their_query_modules() -> None: - expected_modules = { - "list_models": "model_query", - "get_model": "model_query", - "get_model_by_country": "model_query", - "list_model_versions": "model_query", - "get_model_version": "model_query", - "list_variables": "variable_query", - "get_variable": "variable_query", - "list_parameters": "parameter_query", - "get_parameter": "parameter_query", - "list_parameter_children": "parameter_query", - "list_parameter_values": "parameter_query", - "get_parameter_value": "parameter_query", - "list_datasets": "dataset_query", - "get_dataset": "dataset_query", - "list_regions": "region_query", - "get_region": "region_query", - "get_region_by_code": "region_query", - "get_economy_options": "region_query", +def test_resource_entrypoints_are_defined_in_the_service_module() -> None: + method_names = { + "list_models", + "get_model", + "get_model_by_country", + "list_model_versions", + "get_model_version", + "list_variables", + "get_variable", + "list_parameters", + "get_parameter", + "list_parameter_children", + "list_parameter_values", + "get_parameter_value", + "list_datasets", + "get_dataset", + "list_regions", + "get_region", + "get_region_by_code", + "get_economy_options", } - for method_name, module_name in expected_modules.items(): - method = getattr(V2MetadataQueryService, method_name) - assert method.__module__.endswith(f".{module_name}") + for method_name in method_names: + method = getattr(V2MetadataService, method_name) + assert method.__module__.endswith(".services") diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index efbaa5c05..2a4cc854e 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -1,23 +1,65 @@ """Canonical SQLModel persistence and bounded SQLAlchemy escape-hatch tests.""" -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path import pytest +import sqlalchemy as sa from sqlalchemy.exc import IntegrityError from sqlmodel import Session, create_engine, select from policyengine_api.data.v2.models import ( + Dynamic, + LegacyPolicyMapping, + LegacyUserMapping, + LegacyUserPolicyMapping, Parameter, ParameterValue, + Policy, TaxBenefitModel, TaxBenefitModelVersion, User, + UserPolicy, V2_METADATA, ) from policyengine_api.data.v2.models.base import DIRECT_SQLALCHEMY_EXCEPTIONS +def _relational_sqlite_engine(): + engine = create_engine("sqlite://") + + @sa.event.listens_for(engine, "connect") + def enable_foreign_keys(dbapi_connection, _connection_record) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + V2_METADATA.create_all(engine) + return engine + + +def _policy_graph(*, content_hash: str = "a" * 64): + model = TaxBenefitModel(name=f"policy-model-{content_hash[:8]}") + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.example.rate", + tax_benefit_model_version=version, + ) + policy = Policy( + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash=content_hash, + ) + return model, version, parameter, policy + + def test_ordinary_persistence_uses_sqlmodel_session_select_and_exec() -> None: # SQLite exists only as an injected, in-memory unit-test fixture. Runtime # v2 settings reject it and application code never selects it. @@ -121,6 +163,243 @@ def test_canonical_parameter_values_are_unique_by_parameter_and_start_date() -> engine.dispose() +def test_policy_content_hash_is_unique_per_canonicalization_version() -> None: + engine = _relational_sqlite_engine() + model, version, _parameter, policy = _policy_graph() + duplicate = Policy( + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=policy.canonicalization_version, + content_hash=policy.content_hash, + ) + + with Session(engine) as session: + session.add_all([policy, duplicate]) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_policy_parameter_value_owner_period_and_identity_constraints() -> None: + engine = _relational_sqlite_engine() + _model, _version, parameter, policy = _policy_graph(content_hash="b" * 64) + dynamic = Dynamic(name="dynamic") + start = datetime(2026, 1, 1, tzinfo=timezone.utc) + + with Session(engine) as session: + session.add_all([parameter, policy, dynamic]) + session.commit() + + invalid_owner = ParameterValue( + parameter_id=parameter.id, + policy_id=policy.id, + dynamic_id=dynamic.id, + value_json={"rate": 0.1}, + start_date=start, + ) + session.add(invalid_owner) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + invalid_period = ParameterValue( + parameter_id=parameter.id, + policy_id=policy.id, + value_json={"rate": 0.1}, + start_date=start, + end_date=start - timedelta(seconds=1), + ) + session.add(invalid_period) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + session.add_all( + [ + ParameterValue( + parameter_id=parameter.id, + policy_id=policy.id, + value_json={"rate": 0.1}, + start_date=start, + ), + ParameterValue( + parameter_id=parameter.id, + policy_id=policy.id, + value_json={"rate": 0.2}, + start_date=start, + ), + ] + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_user_policy_allows_duplicates_but_requires_policy_country() -> None: + engine = _relational_sqlite_engine() + _model, _version, _parameter, policy = _policy_graph(content_hash="c" * 64) + user = User(primary_country="us") + + with Session(engine) as session: + session.add_all([policy, user]) + session.commit() + first = UserPolicy( + country_id="us", + user_id=user.id, + policy_id=policy.id, + name="First", + ) + second = UserPolicy( + country_id="us", + user_id=user.id, + policy_id=policy.id, + name="Second", + ) + session.add_all([first, second]) + session.commit() + + assert first.id != second.id + + session.add( + UserPolicy( + country_id="uk", + user_id=user.id, + policy_id=policy.id, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_legacy_policy_mapping_allows_many_sources_for_one_policy() -> None: + engine = _relational_sqlite_engine() + _model, _version, _parameter, policy = _policy_graph(content_hash="d" * 64) + + with Session(engine) as session: + session.add(policy) + session.commit() + session.add_all( + [ + LegacyPolicyMapping( + country_id="us", + legacy_policy_id=101, + policy_id=policy.id, + source_policy_hash="1" * 64, + ), + LegacyPolicyMapping( + country_id="us", + legacy_policy_id=102, + policy_id=policy.id, + source_policy_hash="2" * 64, + ), + ] + ) + session.commit() + + assert len(policy.legacy_mappings) == 2 + + session.add( + LegacyPolicyMapping( + country_id="us", + legacy_policy_id=101, + policy_id=policy.id, + source_policy_hash="3" * 64, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_legacy_user_policy_mapping_destination_is_unique_and_cascades() -> None: + engine = _relational_sqlite_engine() + _model, _version, _parameter, policy = _policy_graph(content_hash="e" * 64) + user = User(primary_country="us") + + with Session(engine) as session: + association = UserPolicy( + country_id="us", + user=user, + policy=policy, + ) + mapping = LegacyUserPolicyMapping( + country_id="us", + legacy_user_policy_id=201, + association=association, + fingerprint_version=1, + fingerprint_sha256="4" * 64, + ) + session.add(mapping) + session.commit() + assert mapping.created_at is not None + assert mapping.updated_at is not None + + session.add( + LegacyUserPolicyMapping( + country_id="us", + legacy_user_policy_id=202, + user_policy_id=association.id, + fingerprint_version=1, + fingerprint_sha256="5" * 64, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + session.delete(association) + session.commit() + stored_mapping = session.exec( + select(LegacyUserPolicyMapping).where( + LegacyUserPolicyMapping.id == mapping.id + ) + ).one_or_none() + assert stored_mapping is None + + stored_policy = session.get(Policy, policy.id) + assert stored_policy is not None + + engine.dispose() + + +def test_legacy_user_mapping_is_one_to_one_and_profile_fields_are_optional() -> None: + engine = _relational_sqlite_engine() + + with Session(engine) as session: + user = User(primary_country="us") + session.add(user) + session.commit() + mapping = LegacyUserMapping( + legacy_user_id="legacy-user", + user_id=user.id, + ) + session.add(mapping) + session.commit() + + assert mapping.user_id == user.id + assert user.first_name is None + assert user.last_name is None + assert user.email is None + assert mapping.created_at is not None + + session.add( + LegacyUserMapping( + legacy_user_id="another-legacy-user", + user_id=user.id, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + def test_v2_models_do_not_create_a_parallel_sqlalchemy_orm_layer() -> None: models_directory = ( Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" / "models" diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index dbdab10ee..f22449932 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -11,6 +11,10 @@ Dynamic, Household, HouseholdJob, + LegacyPolicyMapping, + LegacyUserMapping, + LegacyUserPolicyMapping, + ParameterValue, Policy, Simulation, TaxBenefitModelVersion, @@ -50,6 +54,9 @@ def test_domain_models_are_grouped_into_topic_scoped_modules() -> None: Simulation: "simulations", UserHouseholdAssociation: "associations", UserPolicy: "associations", + LegacyPolicyMapping: "policy_mappings", + LegacyUserMapping: "policy_mappings", + LegacyUserPolicyMapping: "policy_mappings", UserReportAssociation: "associations", UserSimulationAssociation: "associations", } @@ -105,7 +112,7 @@ def test_every_declared_relationship_has_a_complete_back_populates_pair() -> Non assert inverse.mapper is mapper -def test_every_user_association_has_relational_integrity() -> None: +def test_user_owned_associations_have_relational_integrity() -> None: configure_mappers() user_mapper = sa.inspect(User) associations = ( @@ -114,12 +121,12 @@ def test_every_user_association_has_relational_integrity() -> None: "user_household_associations", "household_associations", ), - (UserPolicy, "user_policies", "policy_associations"), ( UserSimulationAssociation, "user_simulation_associations", "simulation_associations", ), + (UserPolicy, "user_policies", "policy_associations"), (UserReportAssociation, "user_report_associations", "report_associations"), ) @@ -135,6 +142,146 @@ def test_every_user_association_has_relational_integrity() -> None: assert user_mapper.relationships[user_collection].back_populates == "user" +def test_policy_content_identity_and_catalog_columns_are_explicit() -> None: + policies = V2_METADATA.tables["policies"] + + assert {"name", "description"}.isdisjoint(policies.c.keys()) + assert { + "country_id", + "tax_benefit_model_id", + "tax_benefit_model_version_id", + "canonicalization_version", + "content_hash", + "created_at", + "updated_at", + }.issubset(policies.c.keys()) + assert policies.c.country_id.type.length == 2 + assert policies.c.content_hash.type.length == 64 + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in policies.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("id", "country_id") in unique_columns + assert ("canonicalization_version", "content_hash") in unique_columns + assert { + "ix_policies_country_model", + "ix_policies_country_model_version", + } <= {index.name for index in policies.indexes} + + +def test_policy_parameter_values_use_jsonb_and_enforce_period_identity() -> None: + values = V2_METADATA.tables["parameter_values"] + + postgres_type = values.c.value_json.type.dialect_impl(postgresql.dialect()) + assert isinstance(postgres_type, postgresql.JSONB) + assert { + "ck_parameter_values_single_owner", + "ck_parameter_values_effective_period", + } <= {constraint.name for constraint in values.constraints} + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in values.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("policy_id", "parameter_id", "start_date") in unique_columns + assert sa.inspect(ParameterValue).relationships["policy"].back_populates == ( + "parameter_values" + ) + + +def test_user_policy_is_an_independent_country_scoped_association() -> None: + associations = V2_METADATA.tables["user_policies"] + + assert isinstance(associations.c.user_id.type, sa.Uuid) + user_foreign_key = next(iter(associations.c.user_id.foreign_keys)) + assert user_foreign_key.target_fullname == "users.id" + assert user_foreign_key.ondelete == "CASCADE" + assert {"country", "label"}.isdisjoint(associations.c.keys()) + assert {"country_id", "name", "description"}.issubset(associations.c.keys()) + assert associations.c.name.nullable + assert associations.c.description.nullable + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in associations.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("user_id", "policy_id") not in unique_columns + assert ("id", "country_id") in unique_columns + policy_country = next( + constraint + for constraint in associations.foreign_key_constraints + if constraint.name == "fk_user_policies_policy_country" + ) + assert [column.name for column in policy_country.columns] == [ + "policy_id", + "country_id", + ] + assert [element.target_fullname for element in policy_country.elements] == [ + "policies.id", + "policies.country_id", + ] + assert policy_country.ondelete == "RESTRICT" + assert sa.inspect(UserPolicy).relationships["user"].back_populates == ( + "policy_associations" + ) + + +def test_legacy_user_mapping_is_one_to_one() -> None: + mappings = V2_METADATA.tables["legacy_user_mappings"] + + assert mappings.primary_key.columns.keys() == ["legacy_user_id"] + assert mappings.c.legacy_user_id.type.length == 255 + assert mappings.c.created_at.type.timezone + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in mappings.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("user_id",) in unique_columns + user_foreign_key = next(iter(mappings.c.user_id.foreign_keys)) + assert user_foreign_key.target_fullname == "users.id" + assert user_foreign_key.ondelete == "RESTRICT" + + +def test_legacy_policy_mapping_is_many_to_one_by_destination() -> None: + mappings = V2_METADATA.tables["legacy_policy_mappings"] + + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in mappings.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("country_id", "legacy_policy_id") in unique_columns + assert ("policy_id",) not in unique_columns + assert mappings.c.source_policy_hash.type.length == 255 + policy_country = next(iter(mappings.foreign_key_constraints)) + assert policy_country.name == "fk_legacy_policy_mappings_policy_country" + assert policy_country.ondelete == "RESTRICT" + + +def test_legacy_user_policy_mapping_is_one_to_one_by_destination() -> None: + mappings = V2_METADATA.tables["legacy_user_policy_mappings"] + + unique_columns = { + tuple(column.name for column in constraint.columns) + for constraint in mappings.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("country_id", "legacy_user_policy_id") in unique_columns + assert ("user_policy_id",) in unique_columns + assert mappings.c.fingerprint_sha256.type.length == 64 + assert mappings.c.last_applied_source_revision.server_default.arg == "0" + assert "ck_legacy_user_policy_mappings_source_revision" in { + constraint.name for constraint in mappings.constraints + } + association_country = next(iter(mappings.foreign_key_constraints)) + assert ( + association_country.name == "fk_legacy_user_policy_mappings_association_country" + ) + assert association_country.ondelete == "CASCADE" + + def test_all_datetime_columns_are_timezone_aware() -> None: datetime_columns = [ column @@ -193,6 +340,9 @@ def test_user_primary_country_is_required_and_limited_to_supported_values() -> N users = V2_METADATA.tables["users"] assert not users.c.primary_country.nullable + assert users.c.first_name.nullable + assert users.c.last_name.nullable + assert users.c.email.nullable assert users.c.primary_country.type.length == 2 assert "ck_users_primary_country" in { constraint.name for constraint in users.constraints diff --git a/tests/unit/v2/test_policy_canonicalization.py b/tests/unit/v2/test_policy_canonicalization.py new file mode 100644 index 000000000..3b750456b --- /dev/null +++ b/tests/unit/v2/test_policy_canonicalization.py @@ -0,0 +1,182 @@ +"""Deterministic canonical policy-content tests.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import hashlib +from uuid import UUID, uuid4 + +from policyengine_api.services.v2.policies.transformations import ( + POLICY_CANONICALIZATION_VERSION, + canonical_policy_document, + canonicalize_policy, +) +from policyengine_api.services.v2.policies.types import ( + ResolvedPolicyCreationInput, +) + + +MODEL_ID = UUID("00000000-0000-0000-0000-000000000010") +MODEL_VERSION_ID = UUID("00000000-0000-0000-0000-000000000020") +FIRST_PARAMETER_ID = UUID("00000000-0000-0000-0000-000000000030") +SECOND_PARAMETER_ID = UUID("00000000-0000-0000-0000-000000000040") + + +def _command(*, values=None, **changes) -> ResolvedPolicyCreationInput: + fields = { + "country_id": "us", + "tax_benefit_model_id": MODEL_ID, + "tax_benefit_model_version_id": MODEL_VERSION_ID, + "policyengine_version": "5.2.0", + "parameter_values": values + if values is not None + else [ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"enabled": True, "rate": 1}, + "start_date": "2026-01-01T00:00:00Z", + } + ], + } + fields.update(changes) + return ResolvedPolicyCreationInput.model_validate(fields) + + +def test_document_has_versioned_deterministic_content_only() -> None: + document = canonical_policy_document(_command()) + + assert document == ( + b'{"canonicalization_version":1,"country_id":"us",' + b'"parameter_values":[{"end_date":null,' + b'"parameter_id":"00000000-0000-0000-0000-000000000030",' + b'"start_date":"2026-01-01T00:00:00.000000Z",' + b'"value":{"enabled":true,"rate":1}}],' + b'"tax_benefit_model_id":"00000000-0000-0000-0000-000000000010",' + b'"tax_benefit_model_version_id":' + b'"00000000-0000-0000-0000-000000000020"}' + ) + assert b"policyengine_version" not in document + assert b"created_at" not in document + assert b"name" not in document + + +def test_request_and_object_member_order_do_not_change_identity() -> None: + first = { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"z": [3, 2, 1], "a": {"right": 2, "left": 1}}, + "start_date": "2026-01-01T00:00:00Z", + } + second = { + "parameter_id": SECOND_PARAMETER_ID, + "value": "second", + "start_date": "2027-01-01T00:00:00Z", + } + reordered_first = { + **first, + "value": {"a": {"left": 1, "right": 2}, "z": [3, 2, 1]}, + } + + assert canonicalize_policy(_command(values=[first, second])) == canonicalize_policy( + _command(values=[second, reordered_first]) + ) + + +def test_equivalent_json_numbers_and_utc_instants_have_one_encoding() -> None: + integer = _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"positive": 1, "zero": 0}, + "start_date": "2026-01-01T00:00:00Z", + } + ] + ) + floating = _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"zero": -0.0, "positive": 1.0}, + "start_date": "2026-01-01T03:00:00+03:00", + } + ] + ) + + assert canonicalize_policy(integer) == canonicalize_policy(floating) + + +def test_material_content_changes_produce_distinct_documents() -> None: + original = canonical_policy_document(_command()) + alternatives = [ + _command(country_id="uk"), + _command(tax_benefit_model_id=uuid4()), + _command(tax_benefit_model_version_id=uuid4()), + _command( + values=[ + { + "parameter_id": SECOND_PARAMETER_ID, + "value": {"enabled": True, "rate": 1}, + "start_date": "2026-01-01T00:00:00Z", + } + ] + ), + _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"enabled": True, "rate": 2}, + "start_date": "2026-01-01T00:00:00Z", + } + ] + ), + _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"enabled": True, "rate": 1}, + "start_date": "2026-01-02T00:00:00Z", + } + ] + ), + _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": {"enabled": True, "rate": 1}, + "start_date": "2026-01-01T00:00:00Z", + "end_date": "2026-12-31T00:00:00Z", + } + ] + ), + ] + + assert all(canonical_policy_document(item) != original for item in alternatives) + + +def test_digest_is_sha256_of_exact_canonical_bytes() -> None: + content = canonicalize_policy(_command()) + + assert content.version == POLICY_CANONICALIZATION_VERSION + assert content.content_hash == hashlib.sha256(content.document).hexdigest() + assert len(content.content_hash) == 64 + + +def test_datetime_objects_are_rendered_at_fixed_microsecond_precision() -> None: + command = _command( + values=[ + { + "parameter_id": FIRST_PARAMETER_ID, + "value": 1, + "start_date": datetime( + 2026, + 1, + 1, + 0, + 0, + 0, + 42, + tzinfo=timezone.utc, + ), + } + ] + ) + assert b"2026-01-01T00:00:00.000042Z" in canonical_policy_document(command) diff --git a/tests/unit/v2/test_policy_catalog.py b/tests/unit/v2/test_policy_catalog.py new file mode 100644 index 000000000..3133026dd --- /dev/null +++ b/tests/unit/v2/test_policy_catalog.py @@ -0,0 +1,154 @@ +"""Exact catalog-binding tests for immutable policies.""" + +from __future__ import annotations + +from uuid import uuid4 + +from sqlmodel import Session, create_engine +import pytest + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.data.v2.models import ( + Parameter, + TaxBenefitModel, + TaxBenefitModelVersion, + V2_METADATA, +) +from policyengine_api.services.v2.policies.services import ( + resolve_policy_creation_input, +) +from policyengine_api.services.v2.policies.types import PolicyCreationInput +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, +) + + +def _catalog_session(): + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + session = Session(engine) + us_model = TaxBenefitModel(name="policyengine-us") + current = TaxBenefitModelVersion( + model=us_model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + previous = TaxBenefitModelVersion( + model=us_model, + version="5.1.0", + current_law_id=2, + metadata_time_periods=[2025], + ) + current_parameter = Parameter( + name="gov.example.rate", + tax_benefit_model_version=current, + ) + previous_parameter = Parameter( + name="gov.example.old_rate", + tax_benefit_model_version=previous, + ) + session.add_all([current_parameter, previous_parameter]) + session.commit() + return engine, session, us_model, current, current_parameter, previous_parameter + + +def _command(model_id, parameter_id=None, *, country_id="us"): + parameter_values = [] + if parameter_id is not None: + parameter_values.append( + { + "parameter_id": parameter_id, + "value": 0.2, + "start_date": "2026-01-01T00:00:00Z", + } + ) + return PolicyCreationInput( + country_id=country_id, + tax_benefit_model_id=model_id, + parameter_values=parameter_values, + ) + + +def test_resolver_binds_model_version_and_all_parameter_ids() -> None: + engine, session, model, version, parameter, _previous = _catalog_session() + try: + resolved = resolve_policy_creation_input( + session, + _command(model.id, parameter.id), + policyengine_version="5.2.0", + running_policyengine_version="different-running-version", + ) + + assert resolved.country_id == "us" + assert resolved.tax_benefit_model_id == model.id + assert resolved.tax_benefit_model_version_id == version.id + assert resolved.policyengine_version == "5.2.0" + assert resolved.parameter_values[0].parameter_id == parameter.id + finally: + session.close() + engine.dispose() + + +def test_omitted_version_selects_the_running_catalog() -> None: + engine, session, model, version, _parameter, _previous = _catalog_session() + try: + resolved = resolve_policy_creation_input( + session, + _command(model.id), + running_policyengine_version="5.2.0", + ) + assert resolved.tax_benefit_model_version_id == version.id + finally: + session.close() + engine.dispose() + + +def test_wrong_stable_model_is_rejected() -> None: + engine, session, _model, _version, parameter, _previous = _catalog_session() + try: + with pytest.raises(PolicyCatalogValidationError, match="selected country"): + resolve_policy_creation_input( + session, + _command(uuid4(), parameter.id), + policyengine_version="5.2.0", + ) + finally: + session.close() + engine.dispose() + + +def test_parameter_from_another_model_version_is_rejected() -> None: + engine, session, model, _version, _parameter, previous = _catalog_session() + try: + with pytest.raises(PolicyCatalogValidationError, match="every parameter_id"): + resolve_policy_creation_input( + session, + _command(model.id, previous.id), + policyengine_version="5.2.0", + ) + finally: + session.close() + engine.dispose() + + +def test_absent_or_unsupported_catalog_never_falls_back() -> None: + engine, session, model, _version, _parameter, _previous = _catalog_session() + try: + with pytest.raises(MetadataCatalogVersionNotFoundError): + resolve_policy_creation_input( + session, + _command(model.id), + policyengine_version="4.0.0", + ) + with pytest.raises(MetadataCatalogVersionNotFoundError): + resolve_policy_creation_input( + session, + _command(model.id, country_id="uk"), + policyengine_version="5.2.0", + ) + finally: + session.close() + engine.dispose() diff --git a/tests/unit/v2/test_policy_inputs.py b/tests/unit/v2/test_policy_inputs.py new file mode 100644 index 000000000..9b3153604 --- /dev/null +++ b/tests/unit/v2/test_policy_inputs.py @@ -0,0 +1,148 @@ +"""Validation tests for immutable API v2 policy inputs.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from uuid import uuid4 + +from pydantic import ValidationError +import pytest + +from policyengine_api.services.v2.policies.types import ( + NativePolicyCreationInput, + PolicyCreationInput, + PolicyParameterValueInput, + ResolvedPolicyCreationInput, +) +from policyengine_api.services.v2.policies.validators import ( + MAXIMUM_POLICY_PARAMETER_VALUES, +) + + +def _value(**changes) -> dict[str, object]: + fields: dict[str, object] = { + "parameter_id": uuid4(), + "value": {"rate": 0.2, "enabled": True}, + "start_date": "2026-01-01T03:00:00+03:00", + "end_date": None, + } + fields.update(changes) + return fields + + +def _command(**changes) -> dict[str, object]: + fields: dict[str, object] = { + "country_id": "US", + "tax_benefit_model_id": uuid4(), + "parameter_values": [_value()], + } + fields.update(changes) + return fields + + +def test_input_normalizes_country_and_effective_dates_to_utc() -> None: + policy_input = PolicyCreationInput.model_validate(_command()) + + assert policy_input.country_id == "us" + assert policy_input.parameter_values[0].start_date == datetime( + 2026, + 1, + 1, + tzinfo=timezone.utc, + ) + + +def test_native_and_resolved_inputs_keep_catalog_selection_explicit() -> None: + native = NativePolicyCreationInput.model_validate( + {**_command(), "policyengine_version": "5.2.0"} + ) + resolved = ResolvedPolicyCreationInput.model_validate( + { + **native.model_dump(exclude={"policyengine_version"}), + "policyengine_version": "5.2.0", + "tax_benefit_model_version_id": uuid4(), + } + ) + + assert native.policyengine_version == "5.2.0" + assert resolved.tax_benefit_model_version_id is not None + + +@pytest.mark.parametrize( + "value", + [ + float("nan"), + float("inf"), + float("-inf"), + Decimal("1.2"), + {1: "non-string key"}, + ("tuple",), + object(), + ], +) +def test_non_json_values_are_rejected(value: object) -> None: + with pytest.raises(ValidationError): + PolicyParameterValueInput.model_validate(_value(value=value)) + + +def test_json_reference_cycles_are_rejected() -> None: + cyclic: list[object] = [] + cyclic.append(cyclic) + + with pytest.raises(ValidationError, match="reference cycles"): + PolicyParameterValueInput.model_validate(_value(value=cyclic)) + + +@pytest.mark.parametrize( + "changes", + [ + {"start_date": "2026-01-01T00:00:00"}, + { + "start_date": "2026-01-02T00:00:00Z", + "end_date": "2026-01-01T00:00:00Z", + }, + ], +) +def test_invalid_effective_dates_are_rejected(changes: dict[str, object]) -> None: + with pytest.raises(ValidationError): + PolicyParameterValueInput.model_validate(_value(**changes)) + + +def test_duplicate_parameter_and_normalized_start_date_is_rejected() -> None: + parameter_id = uuid4() + first = _value( + parameter_id=parameter_id, + start_date="2026-01-01T00:00:00Z", + ) + duplicate = _value( + parameter_id=parameter_id, + start_date="2026-01-01T03:00:00+03:00", + ) + + with pytest.raises(ValidationError, match="parameter_id/start_date"): + PolicyCreationInput.model_validate( + _command(parameter_values=[first, duplicate]) + ) + + +def test_parameter_value_count_is_bounded_but_empty_policy_is_valid() -> None: + assert ( + PolicyCreationInput.model_validate( + _command(parameter_values=[]) + ).parameter_values + == [] + ) + + repeated = _value() + parameter_values = [ + { + **repeated, + "parameter_id": uuid4(), + "start_date": datetime(2026, 1, 1, tzinfo=timezone.utc) + + timedelta(days=index), + } + for index in range(MAXIMUM_POLICY_PARAMETER_VALUES + 1) + ] + with pytest.raises(ValidationError): + PolicyCreationInput.model_validate(_command(parameter_values=parameter_values)) diff --git a/tests/unit/v2/test_policy_legacy_translation.py b/tests/unit/v2/test_policy_legacy_translation.py new file mode 100644 index 000000000..bf569616e --- /dev/null +++ b/tests/unit/v2/test_policy_legacy_translation.py @@ -0,0 +1,206 @@ +"""Legacy policy snapshot and translation tests.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from pydantic import ValidationError +from sqlmodel import Session, create_engine +import pytest + +from policyengine_api.data.v2.models import ( + Parameter, + TaxBenefitModel, + TaxBenefitModelVersion, + V2_METADATA, +) +from policyengine_api.services.v2.policies.database_connectors.reads import ( + read_parameters_by_name, + read_policy_catalog, +) +from policyengine_api.services.v2.policies.transformations import ( + parse_legacy_period, + translate_legacy_policy, +) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyTranslationError, +) + + +def _session_and_catalog(): + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + session = Session(engine) + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + first = Parameter( + name="gov.example.rate", + tax_benefit_model_version=version, + ) + second = Parameter( + name="gov.example.amount", + tax_benefit_model_version=version, + ) + session.add_all([first, second]) + session.commit() + return engine, session, model, version, first, second + + +def _snapshot(**changes) -> LegacyPolicySnapshot: + fields = { + "country_id": "us", + "legacy_policy_id": 42, + "label": "Presentation only", + "api_version": "1.0.0", + "policy_json": { + "gov.example.rate": {"2026": 0.2}, + "gov.example.amount": {"2026-01-01.2026-12-31": 100}, + }, + "source_policy_hash": "legacy/base64+hash=", + } + fields.update(changes) + return LegacyPolicySnapshot.model_validate(fields) + + +def _translate( + session: Session, + snapshot: LegacyPolicySnapshot, + *, + running_policyengine_version: str = "5.2.0", +): + selected = read_policy_catalog( + session, + snapshot.country_id, + running_policyengine_version=running_policyengine_version, + ) + assert isinstance(snapshot.policy_json, dict) + parameters = read_parameters_by_name( + session, + model_version_id=selected.model_version.id, + names=set(snapshot.policy_json), + ) + return translate_legacy_policy( + snapshot, + selected=selected, + parameters=parameters, + country_package_versions={"us": "1.0.0"}, + ) + + +def test_year_day_and_explicit_range_periods_are_inclusive_utc() -> None: + assert parse_legacy_period("2026") == ( + datetime(2026, 1, 1, tzinfo=timezone.utc), + datetime(2026, 12, 31, tzinfo=timezone.utc), + ) + assert parse_legacy_period("2026-03-02") == ( + datetime(2026, 3, 2, tzinfo=timezone.utc), + datetime(2026, 3, 2, tzinfo=timezone.utc), + ) + assert parse_legacy_period("2026-02-01.2026-02-28") == ( + datetime(2026, 2, 1, tzinfo=timezone.utc), + datetime(2026, 2, 28, tzinfo=timezone.utc), + ) + + +def test_translation_resolves_paths_and_excludes_legacy_identity_and_label() -> None: + engine, session, model, version, first, second = _session_and_catalog() + try: + translated = _translate(session, _snapshot()) + + assert translated.tax_benefit_model_id == model.id + assert translated.tax_benefit_model_version_id == version.id + assert {value.parameter_id for value in translated.parameter_values} == { + first.id, + second.id, + } + assert "label" not in type(translated).model_fields + assert "legacy_policy_id" not in type(translated).model_fields + finally: + session.close() + engine.dispose() + + +def test_label_does_not_change_translated_core_content() -> None: + engine, session, _model, _version, _first, _second = _session_and_catalog() + try: + first = _translate(session, _snapshot(label="First")) + second = _translate(session, _snapshot(label="Second", legacy_policy_id=43)) + assert first == second + finally: + session.close() + engine.dispose() + + +@pytest.mark.parametrize( + "policy_json", + [ + {"gov.missing": {"2026": 1}}, + {"gov.example.rate": 1}, + {"gov.example.rate": {"not-a-period": 1}}, + { + "gov.example.rate": { + "2026": 1, + "2026-01-01.2026-12-31": 2, + } + }, + ], +) +def test_missing_paths_malformed_periods_and_conflicts_fail( + policy_json: dict[str, object], +) -> None: + engine, session, _model, _version, _first, _second = _session_and_catalog() + try: + with pytest.raises(LegacyPolicyTranslationError): + _translate(session, _snapshot(policy_json=policy_json)) + finally: + session.close() + engine.dispose() + + +def test_country_package_version_must_match_running_release() -> None: + engine, session, _model, _version, _first, _second = _session_and_catalog() + try: + with pytest.raises(LegacyPolicyTranslationError, match="api_version"): + _translate(session, _snapshot(api_version="0.9.0")) + finally: + session.close() + engine.dispose() + + +@pytest.mark.parametrize( + "changes", + [ + {"policy_json": ["not", "an", "object"]}, + {"source_policy_hash": ""}, + {"legacy_policy_id": -1}, + {"policy_json": {"gov.example": {"2026": float("nan")}}}, + ], +) +def test_snapshot_rejects_incomplete_or_non_json_committed_fields(changes) -> None: + with pytest.raises(ValidationError): + _snapshot(**changes) + + +def test_reverse_legacy_range_is_rejected() -> None: + with pytest.raises(LegacyPolicyTranslationError, match="ends before"): + parse_legacy_period("2026-12-31.2026-01-01") + + +def test_unknown_policyengine_version_never_falls_back() -> None: + engine, session, _model, _version, _first, _second = _session_and_catalog() + try: + with pytest.raises(Exception, match="running PolicyEngine.py"): + _translate( + session, + _snapshot(), + running_policyengine_version="4.0.0", + ) + finally: + session.close() + engine.dispose() diff --git a/tests/unit/v2/test_policy_migration_qualification.py b/tests/unit/v2/test_policy_migration_qualification.py new file mode 100644 index 000000000..cc3f63201 --- /dev/null +++ b/tests/unit/v2/test_policy_migration_qualification.py @@ -0,0 +1,174 @@ +"""Tests for read-only v2 policy migration qualification.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +import pytest + +from policyengine_api.data.v2 import policy_migration_qualification as qualification +from policyengine_api.data.v2.settings import ( + V2_MIGRATION_DATABASE_URL, + V2_SUPABASE_ENVIRONMENT, + V2_SUPABASE_PROJECT_REF, +) + + +ENVIRONMENT = { + V2_MIGRATION_DATABASE_URL: ( + "postgresql+psycopg://migration:test-password@db." + "abcdefghijklmnopqrst.supabase.co/postgres?sslmode=require" + ), + V2_SUPABASE_PROJECT_REF: "abcdefghijklmnopqrst", + V2_SUPABASE_ENVIRONMENT: "staging", +} + + +class FakeTransaction: + def __init__(self) -> None: + self.rolled_back = False + + def rollback(self) -> None: + self.rolled_back = True + + +class FakeConnection: + def __init__(self, counts: tuple[int, int, int]) -> None: + self._counts = iter(counts) + self.transaction = FakeTransaction() + self.statements: list[str] = [] + + def begin(self) -> FakeTransaction: + return self.transaction + + def exec_driver_sql(self, statement: str) -> None: + self.statements.append(statement) + + def scalar(self, _statement: object) -> int: + return next(self._counts) + + +class FakeEngine: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + self.disposed = False + + @contextmanager + def connect(self) -> Iterator[FakeConnection]: + yield self.connection + + def dispose(self) -> None: + self.disposed = True + + +def test_empty_target_is_qualified_in_a_rolled_back_read_only_transaction() -> None: + connection = FakeConnection((0, 0, 0)) + engine = FakeEngine(connection) + + evidence = qualification.qualify_policy_migration_target( + ENVIRONMENT, + engine_builder=lambda _settings: engine, + ) + + assert evidence.as_dict() == { + "outcome": "ok", + "qualification": "performed", + "environment": "staging", + "project_ref": "abcdefghijklmnopqrst", + "counts": { + "policies": 0, + "policy_parameter_values": 0, + "user_policies": 0, + }, + } + assert connection.statements == ["SET TRANSACTION READ ONLY"] + assert connection.transaction.rolled_back + assert engine.disposed + + +def test_applied_policy_revision_skips_predecessor_row_qualification() -> None: + connection = FakeConnection((1, 2, 3)) + engine = FakeEngine(connection) + + evidence = qualification.qualify_policy_migration_if_pending( + ENVIRONMENT, + engine_builder=lambda _settings: engine, + pending_checker=lambda _connection: False, + ) + + assert evidence.as_dict() == { + "outcome": "ok", + "qualification": "not-required", + "environment": "staging", + "project_ref": "abcdefghijklmnopqrst", + "counts": { + "policies": 0, + "policy_parameter_values": 0, + "user_policies": 0, + }, + } + assert connection.statements == ["SET TRANSACTION READ ONLY"] + assert connection.transaction.rolled_back + assert engine.disposed + + +def test_pending_policy_revision_runs_predecessor_row_qualification() -> None: + connection = FakeConnection((0, 0, 0)) + engine = FakeEngine(connection) + + evidence = qualification.qualify_policy_migration_if_pending( + ENVIRONMENT, + engine_builder=lambda _settings: engine, + pending_checker=lambda _connection: True, + ) + + assert evidence.required is True + assert connection.statements == ["SET TRANSACTION READ ONLY"] + assert connection.transaction.rolled_back + assert engine.disposed + + +@pytest.mark.parametrize( + "counts", + [ + (1, 0, 0), + (0, 1, 0), + (0, 0, 1), + (1, 2, 3), + ], +) +def test_retained_policy_data_stops_without_committing( + counts: tuple[int, int, int], +) -> None: + connection = FakeConnection(counts) + engine = FakeEngine(connection) + + with pytest.raises( + qualification.RetainedPolicyDataError, + match="migration stopped without modifying data", + ) as raised: + qualification.qualify_policy_migration_target( + ENVIRONMENT, + engine_builder=lambda _settings: engine, + ) + + assert raised.value.counts == qualification.PolicyDataCounts(*counts) + assert connection.transaction.rolled_back + assert engine.disposed + + +def test_main_redacts_unexpected_errors( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def fail() -> None: + raise RuntimeError("postgresql://user:secret@private-host/database") + + monkeypatch.setattr(qualification, "qualify_policy_migration_if_pending", fail) + + assert qualification.main() == 1 + captured = capsys.readouterr() + assert "qualification failed unexpectedly" in captured.err + assert "secret" not in captured.err + assert "private-host" not in captured.err diff --git a/tests/unit/v2/test_policy_persistence_statements.py b/tests/unit/v2/test_policy_persistence_statements.py new file mode 100644 index 000000000..6e03a11cf --- /dev/null +++ b/tests/unit/v2/test_policy_persistence_statements.py @@ -0,0 +1,26 @@ +"""Unit checks for conflict-aware policy insertion statements.""" + +from __future__ import annotations + +from sqlalchemy.dialects import postgresql + +from policyengine_api.services.v2.policies.database_connectors import creates + + +def test_policy_insert_uses_the_content_identity_constraint_and_returning() -> None: + source = creates.create_policy.__code__.co_consts + statement_text = " ".join(str(value) for value in source) + + assert "uq_policies_canonicalization_content_hash" in statement_text + + # Compile a representative statement through the same PostgreSQL dialect + # construct to prove this module does not use a read-before-write insert. + statement = ( + creates.insert(creates.Policy) + .on_conflict_do_nothing(constraint="uq_policies_canonicalization_content_hash") + .returning(creates.Policy.id) + ) + compiled = str(statement.compile(dialect=postgresql.dialect())) + assert "ON CONFLICT ON CONSTRAINT" in compiled + assert "DO NOTHING" in compiled + assert "RETURNING policies.id" in compiled diff --git a/tests/unit/v2/test_policy_query.py b/tests/unit/v2/test_policy_query.py new file mode 100644 index 000000000..ebc9748cd --- /dev/null +++ b/tests/unit/v2/test_policy_query.py @@ -0,0 +1,192 @@ +"""Country-scoped complete policy read tests.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from sqlmodel import Session, create_engine +import pytest + +from policyengine_api.data.v2.models import ( + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + V2_METADATA, +) +from policyengine_api.services.v2.policies.services import ( + read_complete_policy, + read_policy_page, +) +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, +) + + +def _stored_policies(): + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + session = Session(engine) + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + alpha = Parameter(name="gov.alpha", tax_benefit_model_version=version) + zeta = Parameter(name="gov.zeta", tax_benefit_model_version=version) + created = datetime(2026, 1, 1, tzinfo=timezone.utc) + first = Policy( + id=UUID("00000000-0000-0000-0000-000000000010"), + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="1" * 64, + created_at=created, + updated_at=created, + ) + second = Policy( + id=UUID("00000000-0000-0000-0000-000000000020"), + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="2" * 64, + created_at=created + timedelta(seconds=1), + updated_at=created + timedelta(seconds=1), + ) + other_country = Policy( + id=UUID("00000000-0000-0000-0000-000000000030"), + country_id="uk", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="3" * 64, + created_at=created + timedelta(seconds=2), + updated_at=created + timedelta(seconds=2), + ) + session.add_all( + [ + first, + second, + other_country, + ParameterValue( + id=UUID("00000000-0000-0000-0000-000000000200"), + policy=first, + parameter=zeta, + value_json=2, + start_date=created, + ), + ParameterValue( + id=UUID("00000000-0000-0000-0000-000000000100"), + policy=first, + parameter=alpha, + value_json=1, + start_date=created + timedelta(days=1), + ), + ParameterValue( + id=UUID("00000000-0000-0000-0000-000000000090"), + policy=first, + parameter=alpha, + value_json=0, + start_date=created, + ), + ] + ) + session.commit() + return engine, session, model, first, second, other_country + + +def test_detail_joins_parameter_names_and_orders_complete_values() -> None: + engine, session, _model, first, _second, _other = _stored_policies() + try: + result = read_complete_policy(session, country_id="us", policy_id=first.id) + + assert result.id == first.id + assert result.created_at == first.created_at + assert [value.parameter_name for value in result.parameter_values] == [ + "gov.alpha", + "gov.alpha", + "gov.zeta", + ] + assert [value.value for value in result.parameter_values] == [0, 1, 2] + finally: + session.close() + engine.dispose() + + +def test_detail_uses_country_as_part_of_resource_identity() -> None: + engine, session, _model, first, _second, _other = _stored_policies() + try: + with pytest.raises(PolicyNotFoundError): + read_complete_policy(session, country_id="uk", policy_id=first.id) + finally: + session.close() + engine.dispose() + + +def test_empty_policy_has_an_empty_nested_collection() -> None: + engine, session, _model, _first, second, _other = _stored_policies() + try: + assert ( + read_complete_policy( + session, + country_id="us", + policy_id=second.id, + ).parameter_values + == () + ) + finally: + session.close() + engine.dispose() + + +def test_collection_filters_orders_paginates_and_returns_complete_items() -> None: + engine, session, model, first, second, _other = _stored_policies() + try: + first_page = read_policy_page( + session, + country_id="us", + tax_benefit_model_id=model.id, + offset=0, + limit=1, + ) + second_page = read_policy_page( + session, + country_id="us", + tax_benefit_model_id=model.id, + offset=1, + limit=1, + ) + + assert [item.id for item in first_page.items] == [first.id] + assert len(first_page.items[0].parameter_values) == 3 + assert first_page.has_more is True + assert (first_page.offset, first_page.limit) == (0, 1) + assert [item.id for item in second_page.items] == [second.id] + assert second_page.has_more is False + assert all(item.country_id == "us" for item in first_page.items) + finally: + session.close() + engine.dispose() + + +def test_model_filter_is_exact() -> None: + engine, session, _model, _first, _second, _other = _stored_policies() + try: + result = read_policy_page( + session, + country_id="us", + tax_benefit_model_id=UUID("ffffffff-ffff-ffff-ffff-ffffffffffff"), + offset=0, + limit=100, + ) + assert result.items == () + assert result.has_more is False + finally: + session.close() + engine.dispose() diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py new file mode 100644 index 000000000..900de1abd --- /dev/null +++ b/tests/unit/v2/test_policy_routes.py @@ -0,0 +1,429 @@ +"""Native FastAPI contract tests for immutable v2 policies.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import UUID, uuid4 + +from fastapi.testclient import TestClient +from flask import Flask, jsonify +import pytest +from sqlalchemy.exc import OperationalError, TimeoutError + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.services.v2.policies.types import ( + NativePolicyCreation, + PolicyPage, + PolicyParameterValueRead, + PolicyRead, +) +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, + PolicyContentHashCollisionError, + PolicyCreationIntegrityError, + PolicyNotFoundError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) + + +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +MODEL_ID = UUID("00000000-0000-0000-0000-000000000020") +MODEL_VERSION_ID = UUID("00000000-0000-0000-0000-000000000030") +PARAMETER_ID = UUID("00000000-0000-0000-0000-000000000040") +VALUE_ID = UUID("00000000-0000-0000-0000-000000000050") +CREATED_AT = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _policy_read(*, policy_id: UUID = POLICY_ID, country_id: str = "us") -> PolicyRead: + return PolicyRead( + id=policy_id, + country_id=country_id, + tax_benefit_model_id=MODEL_ID, + tax_benefit_model_version_id=MODEL_VERSION_ID, + created_at=CREATED_AT, + updated_at=CREATED_AT + timedelta(seconds=1), + parameter_values=( + PolicyParameterValueRead( + id=VALUE_ID, + parameter_id=PARAMETER_ID, + parameter_name="gov.example.rate", + value={"rate": 0.2}, + start_date=CREATED_AT, + end_date=None, + ), + ), + ) + + +class FakePolicyService: + def __init__(self) -> None: + self.created = True + self.error: Exception | None = None + self.calls: list[tuple[str, object]] = [] + + def _raise(self) -> None: + if self.error is not None: + raise self.error + + def create_policy(self, command) -> NativePolicyCreation: + self.calls.append(("create_policy", command)) + self._raise() + return NativePolicyCreation(item=_policy_read(), created=self.created) + + def get_policy(self, **filters) -> PolicyRead: + self.calls.append(("get_policy", filters)) + self._raise() + return _policy_read(policy_id=filters["policy_id"]) + + def list_policies(self, **filters) -> PolicyPage: + self.calls.append(("list_policies", filters)) + self._raise() + return PolicyPage( + items=(_policy_read(),), + offset=filters["offset"], + limit=filters["limit"], + has_more=False, + ) + + +def _client( + service: FakePolicyService, +) -> tuple[TestClient, dict[str, int]]: + flask_calls = {"count": 0} + flask_app = Flask(__name__) + + @flask_app.route("/", methods=["GET", "POST", "PATCH", "DELETE"]) + def fallback(resource: str): + flask_calls["count"] += 1 + return jsonify({"source": "flask", "resource": resource}) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=lambda: {}, + v2_policy_service_factory=lambda: service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + return ( + TestClient( + create_asgi_app( + flask_app, + dependencies=dependencies, + route_settings=settings, + ), + raise_server_exceptions=False, + ), + flask_calls, + ) + + +def _body(**changes) -> dict[str, object]: + body: dict[str, object] = { + "country_id": "us", + "tax_benefit_model_id": str(MODEL_ID), + "parameter_values": [ + { + "parameter_id": str(PARAMETER_ID), + "value": {"rate": 0.2}, + "start_date": "2026-01-01T00:00:00Z", + } + ], + } + body.update(changes) + return body + + +def test_create_returns_201_for_new_and_200_for_deduplicated_content() -> None: + service = FakePolicyService() + client, flask_calls = _client(service) + + created = client.post( + "/v2/policies?country_id=US&policyengine_version=5.2.0", + json=_body(), + ) + service.created = False + deduplicated = client.post("/v2/policies?country_id=us", json=_body()) + + assert created.status_code == 201 + assert deduplicated.status_code == 200 + assert created.json() == deduplicated.json() + assert created.json() == { + "status": "ok", + "message": None, + "result": { + "item": { + "id": str(POLICY_ID), + "country_id": "us", + "tax_benefit_model_id": str(MODEL_ID), + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:01Z", + "parameter_values": [ + { + "id": str(VALUE_ID), + "parameter_id": str(PARAMETER_ID), + "parameter_name": "gov.example.rate", + "value": {"rate": 0.2}, + "start_date": "2026-01-01T00:00:00Z", + "end_date": None, + } + ], + } + }, + } + first_command = service.calls[0][1] + assert first_command.country_id == "us" + assert first_command.policyengine_version == "5.2.0" + assert flask_calls["count"] == 0 + + +def test_create_rejects_country_mismatch_and_core_presentation_fields() -> None: + service = FakePolicyService() + client, _flask_calls = _client(service) + + mismatch = client.post("/v2/policies?country_id=uk", json=_body()) + named = client.post( + "/v2/policies?country_id=us", + json={**_body(), "name": "Not core content", "description": "No"}, + ) + + assert mismatch.status_code == 400 + assert named.status_code == 422 + assert named.json() == { + "status": "error", + "message": "Invalid API v2 request", + } + assert service.calls == [] + + +def test_create_rejects_unknown_duplicate_and_oversized_input_before_service() -> None: + service = FakePolicyService() + client, _flask_calls = _client(service) + + unknown = client.post( + "/v2/policies?country_id=us&search=rate", + json=_body(), + ) + duplicate = client.post( + "/v2/policies?country_id=us&country_id=uk", + json=_body(), + ) + excessive_values = client.post( + "/v2/policies?country_id=us", + json=_body( + parameter_values=[ + { + "parameter_id": str(uuid4()), + "value": index, + "start_date": "2026-01-01T00:00:00Z", + } + for index in range(1_001) + ] + ), + ) + oversized = client.post( + "/v2/policies?country_id=us", + content=b'{"country_id":"us","padding":"' + b"x" * 1_048_576 + b'"}', + headers={"content-type": "application/json"}, + ) + + assert unknown.status_code == 422 + assert unknown.json() == { + "status": "error", + "message": "Invalid API v2 request", + } + assert duplicate.status_code == 422 + assert excessive_values.status_code == 422 + assert oversized.status_code == 413 + assert oversized.json()["status"] == "error" + assert service.calls == [] + + +def test_detail_is_country_scoped_and_returns_typed_not_found() -> None: + service = FakePolicyService() + client, flask_calls = _client(service) + + response = client.get(f"/v2/policies/{POLICY_ID}?country_id=US") + service.error = PolicyNotFoundError("policy was not found") + missing = client.get(f"/v2/policies/{uuid4()}?country_id=uk") + + assert response.status_code == 200 + assert ( + response.json()["result"]["item"]["parameter_values"][0]["parameter_name"] + == "gov.example.rate" + ) + assert service.calls[0] == ( + "get_policy", + {"country_id": "us", "policy_id": POLICY_ID}, + ) + assert missing.status_code == 404 + assert missing.json() == {"status": "error", "message": "policy was not found"} + assert flask_calls["count"] == 0 + + +def test_list_passes_exact_model_filter_and_canonical_pagination() -> None: + service = FakePolicyService() + client, _flask_calls = _client(service) + + response = client.get( + f"/v2/policies?country_id=us&tax_benefit_model_id={MODEL_ID}&offset=2&limit=3" + ) + search = client.get("/v2/policies?country_id=us&search=rate") + excessive = client.get("/v2/policies?country_id=us&limit=501") + + assert response.status_code == 200 + assert response.json()["result"]["offset"] == 2 + assert response.json()["result"]["limit"] == 3 + assert service.calls == [ + ( + "list_policies", + { + "country_id": "us", + "tax_benefit_model_id": MODEL_ID, + "offset": 2, + "limit": 3, + }, + ) + ] + assert search.status_code == 422 + assert excessive.status_code == 422 + + +@pytest.mark.parametrize( + ("error", "status", "message"), + [ + (PolicyCatalogValidationError("bad model"), 400, "bad model"), + (MetadataCatalogVersionNotFoundError("absent catalog"), 404, "absent"), + ( + PolicyContentHashCollisionError("database statement secret"), + 409, + "conflicts", + ), + ( + PolicyCreationIntegrityError("database statement secret"), + 500, + "integrity", + ), + (V2ConfigurationError("postgresql://secret"), 503, "unavailable"), + ( + MetadataCatalogUnavailableError("database statement secret"), + 503, + "unavailable", + ), + ( + OperationalError("statement secret", {}, Exception("credential")), + 503, + "unavailable", + ), + (TimeoutError("pool timeout"), 503, "unavailable"), + (RuntimeError("credential and policy value"), 500, "operation failed"), + ], +) +def test_policy_failures_map_to_secret_safe_typed_errors( + error: Exception, + status: int, + message: str, +) -> None: + service = FakePolicyService() + service.error = error + client, _flask_calls = _client(service) + + response = client.get(f"/v2/policies/{POLICY_ID}?country_id=us") + + assert response.status_code == status + assert response.json()["status"] == "error" + assert message in response.json()["message"].lower() + assert "secret" not in response.text + assert "credential" not in response.text + + +@pytest.mark.parametrize("method", ["put", "patch", "delete"]) +def test_core_policy_mutations_are_not_exposed(method: str) -> None: + service = FakePolicyService() + client, flask_calls = _client(service) + + response = client.request( + method.upper(), + f"/v2/policies/{POLICY_ID}?country_id=us", + json={}, + ) + + assert response.status_code == 405 + assert response.json()["status"] == "error" + assert service.calls == [] + assert flask_calls["count"] == 0 + + +def test_openapi_publishes_request_query_response_and_error_contracts() -> None: + service = FakePolicyService() + client, _flask_calls = _client(service) + schema = client.get("/v2/openapi.json").json() + + assert set(path for path in schema["paths"] if path.startswith("/v2/policies")) == { + "/v2/policies", + "/v2/policies/{policy_id}", + } + collection = schema["paths"]["/v2/policies"]["get"] + parameters = {item["name"]: item for item in collection["parameters"]} + assert parameters["country_id"]["required"] is True + assert parameters["offset"]["schema"]["default"] == 0 + assert parameters["limit"]["schema"]["default"] == 100 + assert parameters["limit"]["schema"]["maximum"] == 500 + assert parameters["tax_benefit_model_id"]["schema"]["anyOf"][0]["format"] == ( + "uuid" + ) + + create = schema["paths"]["/v2/policies"]["post"] + create_parameters = {item["name"]: item for item in create["parameters"]} + assert create_parameters["country_id"]["required"] is True + assert create_parameters["policyengine_version"]["required"] is False + assert {"200", "201", "400", "404", "409", "413", "422", "500", "503"} <= ( + set(create["responses"]) + ) + request_ref = create["requestBody"]["content"]["application/json"]["schema"]["$ref"] + request_schema = schema["components"]["schemas"][request_ref.rsplit("/", 1)[-1]] + assert request_schema["additionalProperties"] is False + parameter_values = request_schema["properties"]["parameter_values"] + assert parameter_values["maxItems"] == 1000 + item_schema = schema["components"]["schemas"]["PolicyItem"] + assert set(item_schema["required"]) == { + "id", + "country_id", + "tax_benefit_model_id", + "created_at", + "updated_at", + "parameter_values", + } + assert "name" not in item_schema["properties"] + assert "description" not in item_schema["properties"] + + +def test_native_policy_routes_do_not_use_cloud_sql_or_flask( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_cloud_sql(): + raise AssertionError("Cloud SQL must not be selected") + + monkeypatch.setattr( + "policyengine_api.data.orm.get_v1_session_factory", + reject_cloud_sql, + ) + service = FakePolicyService() + client, flask_calls = _client(service) + + assert client.get(f"/v2/policies/{POLICY_ID}?country_id=us").status_code == 200 + assert service.calls[0][0] == "get_policy" + assert flask_calls["count"] == 0 diff --git a/tests/unit/v2/test_settings.py b/tests/unit/v2/test_settings.py index 020aa4f13..f093fd275 100644 --- a/tests/unit/v2/test_settings.py +++ b/tests/unit/v2/test_settings.py @@ -207,6 +207,20 @@ def test_pooler_username_can_identify_the_configured_supabase_project() -> None: assert settings.connection.url.username == f"data-writer.{PROJECT_REF}" +def test_runtime_rejects_transaction_pooling_without_session_timeouts() -> None: + with pytest.raises(V2ConfigurationError, match="session mode on port 5432"): + load_v2_runtime_database_settings( + { + **TARGET_ENVIRONMENT, + V2_RUNTIME_DATABASE_URL: ( + f"postgresql+psycopg://runtime.{PROJECT_REF}:password@" + "aws-0-us-east-2.pooler.supabase.com:6543/" + "postgres?sslmode=require" + ), + } + ) + + def test_supabase_database_name_must_be_postgres() -> None: with pytest.raises(V2ConfigurationError, match="configured Supabase database"): load_v2_runtime_database_settings( diff --git a/tests/unit/v2/test_user_policy_legacy.py b/tests/unit/v2/test_user_policy_legacy.py new file mode 100644 index 000000000..621ff217e --- /dev/null +++ b/tests/unit/v2/test_user_policy_legacy.py @@ -0,0 +1,198 @@ +"""Legacy saved-policy fingerprint and projection command tests.""" + +from __future__ import annotations + +from unittest.mock import MagicMock +from uuid import UUID + +import pytest + +from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy +from policyengine_api.services.v2.user_policies.services import ( + apply_existing_legacy_user_policy_mapping, +) +from policyengine_api.services.v2.user_policies.transformations import ( + USER_POLICY_FINGERPRINT_VERSION, + fingerprint_legacy_user_policy, +) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot +from policyengine_api.services.v2.user_policies.validators import ( + LegacyUserPolicyIntegrityError, +) + + +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +USER_ID = UUID("00000000-0000-0000-0000-000000000070") + + +def _snapshot(**changes) -> LegacyUserPolicySnapshot: + values = { + "country_id": "us", + "legacy_user_policy_id": 10, + "reform_id": 2, + "reform_label": "Reform", + "baseline_id": 1, + "baseline_label": "Current law", + "user_id": "auth0|one", + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 3, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + values.update(changes) + return LegacyUserPolicySnapshot.model_validate(values) + + +def test_complete_row_fingerprint_is_deterministic_and_sha256() -> None: + first = _snapshot() + reordered = LegacyUserPolicySnapshot.model_validate( + dict(reversed(list(first.model_dump().items()))) + ) + + assert fingerprint_legacy_user_policy(first) == fingerprint_legacy_user_policy( + reordered + ) + assert len(fingerprint_legacy_user_policy(first)) == 64 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("reform_label", "Updated"), + ("baseline_label", "Baseline updated"), + ("year", "2027"), + ("geography", "ca"), + ("dataset", None), + ("number_of_provisions", 4), + ("api_version", "2.0.0"), + ("added_date", 3), + ("updated_date", 4), + ("budgetary_impact", "100"), + ("type", "reform"), + ], +) +def test_every_mutable_or_v1_only_field_changes_the_fingerprint( + field: str, + value: object, +) -> None: + assert fingerprint_legacy_user_policy(_snapshot()) != ( + fingerprint_legacy_user_policy(_snapshot(**{field: value})) + ) + + +def test_snapshot_rejects_unknown_or_unbounded_fields() -> None: + with pytest.raises(ValueError): + _snapshot(unexpected="value") + with pytest.raises(ValueError): + _snapshot(user_id="x" * 256) + + +def _apply_update( + *, + changed_fields: frozenset[str], + stored_revision: int = 0, + source_revision: int = 1, + stored_fingerprint: str = "a" * 64, + fingerprint: str = "b" * 64, +): + association = UserPolicy( + country_id="us", + user_id=USER_ID, + policy_id=POLICY_ID, + name="Native name", + description="Native description", + ) + mapping = LegacyUserPolicyMapping( + country_id="us", + legacy_user_policy_id=10, + user_policy_id=association.id, + last_applied_source_revision=stored_revision, + fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, + fingerprint_sha256=stored_fingerprint, + ) + session = MagicMock() + session.exec.return_value.one_or_none.return_value = association + result = apply_existing_legacy_user_policy_mapping( + session, + mapping=mapping, + country_id="us", + reform_label="Legacy rename", + fingerprint=fingerprint, + user_id=USER_ID, + policy_id=POLICY_ID, + changed_fields=changed_fields, + source_revision=source_revision, + ) + return association, mapping, result + + +def test_v1_only_update_advances_fingerprint_without_changing_presentation() -> None: + association, mapping, result = _apply_update( + changed_fields=frozenset({"year", "updated_date"}) + ) + + assert result.association_updated is False + assert association.name == "Native name" + assert association.description == "Native description" + assert mapping.fingerprint_sha256 == "b" * 64 + assert mapping.last_applied_source_revision == 1 + + +def test_reform_label_update_changes_only_name() -> None: + association, mapping, result = _apply_update( + changed_fields=frozenset({"reform_label", "updated_date"}) + ) + + assert result.association_updated is True + assert association.name == "Legacy rename" + assert association.description == "Native description" + assert mapping.fingerprint_sha256 == "b" * 64 + assert mapping.last_applied_source_revision == 1 + + +def test_same_revision_and_fingerprint_is_an_idempotent_replay() -> None: + association, mapping, result = _apply_update( + changed_fields=frozenset({"reform_label"}), + stored_revision=3, + source_revision=3, + stored_fingerprint="b" * 64, + ) + + assert result.association_updated is False + assert association.name == "Native name" + assert mapping.last_applied_source_revision == 3 + + +def test_same_revision_with_different_fingerprint_is_rejected() -> None: + with pytest.raises(LegacyUserPolicyIntegrityError, match="revision conflicts"): + _apply_update( + changed_fields=frozenset({"reform_label"}), + stored_revision=3, + source_revision=3, + ) + + +def test_stale_revision_is_an_idempotent_no_op() -> None: + association, mapping, result = _apply_update( + changed_fields=frozenset({"reform_label"}), + stored_revision=3, + source_revision=2, + ) + + assert result.association_updated is False + assert association.name == "Native name" + assert mapping.last_applied_source_revision == 3 + + +def test_revision_with_an_unapplied_predecessor_is_rejected() -> None: + with pytest.raises(LegacyUserPolicyIntegrityError, match="unapplied predecessor"): + _apply_update( + changed_fields=frozenset({"reform_label"}), + stored_revision=1, + source_revision=3, + ) diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py new file mode 100644 index 000000000..922a99e08 --- /dev/null +++ b/tests/unit/v2/test_user_policy_routes.py @@ -0,0 +1,428 @@ +"""Native FastAPI contract tests for v2 user-policy associations.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import UUID, uuid4 + +from fastapi.testclient import TestClient +from flask import Flask, jsonify +import pytest +from sqlalchemy.exc import OperationalError, SQLAlchemyError, TimeoutError + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.services.v2.user_policies.types import ( + UserPolicyPage, + UserPolicyRead, +) +from policyengine_api.services.v2.user_policies.validators import ( + AssociationCountryConflictError, + AssociationPolicyNotFoundError, + AssociationUserNotFoundError, + UserPolicyNotFoundError, +) +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) + + +ASSOCIATION_ID = UUID("00000000-0000-0000-0000-000000000060") +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +USER_ID = UUID("00000000-0000-0000-0000-000000000070") +CREATED_AT = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _association_read( + *, + association_id: UUID = ASSOCIATION_ID, + country_id: str = "us", + user_id: UUID = USER_ID, + policy_id: UUID = POLICY_ID, + name: str | None = "Saved reform", + description: str | None = "Personal note", +) -> UserPolicyRead: + return UserPolicyRead( + id=association_id, + country_id=country_id, + user_id=user_id, + policy_id=policy_id, + name=name, + description=description, + created_at=CREATED_AT, + updated_at=CREATED_AT + timedelta(seconds=1), + ) + + +class FakeUserPolicyService: + def __init__(self) -> None: + self.error: Exception | None = None + self.calls: list[tuple[str, object]] = [] + self.next_item = _association_read() + + def _raise(self) -> None: + if self.error is not None: + raise self.error + + def create_user_policy(self, association_input) -> UserPolicyRead: + self.calls.append(("create_user_policy", association_input)) + self._raise() + return self.next_item + + def get_user_policy(self, **identity) -> UserPolicyRead: + self.calls.append(("get_user_policy", identity)) + self._raise() + return self.next_item + + def list_user_policies(self, **filters) -> UserPolicyPage: + self.calls.append(("list_user_policies", filters)) + self._raise() + return UserPolicyPage( + items=(self.next_item,), + offset=filters["offset"], + limit=filters["limit"], + has_more=False, + ) + + def patch_user_policy(self, **changes) -> UserPolicyRead: + self.calls.append(("patch_user_policy", changes)) + self._raise() + association_input = changes["association_input"] + name = ( + association_input.name + if "name" in association_input.model_fields_set + else "Saved reform" + ) + description = ( + association_input.description + if "description" in association_input.model_fields_set + else "Personal note" + ) + return _association_read(name=name, description=description) + + def delete_user_policy(self, **identity) -> None: + self.calls.append(("delete_user_policy", identity)) + self._raise() + + +def _client( + service: FakeUserPolicyService, +) -> tuple[TestClient, dict[str, int]]: + flask_calls = {"count": 0} + flask_app = Flask(__name__) + + @flask_app.route("/", methods=["GET", "POST", "PATCH", "DELETE"]) + def fallback(resource: str): + flask_calls["count"] += 1 + return jsonify({"source": "flask", "resource": resource}) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=lambda: {}, + v2_user_policy_service_factory=lambda: service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + return ( + TestClient( + create_asgi_app( + flask_app, + dependencies=dependencies, + route_settings=settings, + ), + raise_server_exceptions=False, + ), + flask_calls, + ) + + +def _body(**changes) -> dict[str, object]: + body: dict[str, object] = { + "country_id": "us", + "user_id": str(USER_ID), + "policy_id": str(POLICY_ID), + "name": "Saved reform", + "description": "Personal note", + } + body.update(changes) + return body + + +def test_create_returns_complete_distinct_association_contract() -> None: + service = FakeUserPolicyService() + client, flask_calls = _client(service) + + response = client.post("/v2/user-policies?country_id=US", json=_body()) + + assert response.status_code == 201 + assert response.json() == { + "status": "ok", + "message": None, + "result": { + "item": { + "id": str(ASSOCIATION_ID), + "country_id": "us", + "user_id": str(USER_ID), + "policy_id": str(POLICY_ID), + "name": "Saved reform", + "description": "Personal note", + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:01Z", + } + }, + } + command = service.calls[0][1] + assert command.user_id == USER_ID + assert command.policy_id == POLICY_ID + assert flask_calls["count"] == 0 + + +def test_repeated_create_calls_service_twice_without_link_deduplication() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + + first = client.post("/v2/user-policies?country_id=us", json=_body()) + service.next_item = _association_read(association_id=uuid4()) + second = client.post("/v2/user-policies?country_id=us", json=_body()) + + assert first.status_code == second.status_code == 201 + assert first.json()["result"]["item"]["id"] != second.json()["result"]["item"]["id"] + assert [call[0] for call in service.calls] == [ + "create_user_policy", + "create_user_policy", + ] + + +def test_create_rejects_country_mismatch_and_invalid_fields() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + + mismatch = client.post("/v2/user-policies?country_id=uk", json=_body()) + invalid_user = client.post( + "/v2/user-policies?country_id=us", + json=_body(user_id="not-a-uuid"), + ) + long_name = client.post( + "/v2/user-policies?country_id=us", + json=_body(name="x" * 256), + ) + + assert mismatch.status_code == 400 + assert invalid_user.status_code == 422 + assert invalid_user.json() == { + "status": "error", + "message": "Invalid API v2 request", + } + assert long_name.status_code == 422 + assert service.calls == [] + + +def test_detail_and_list_use_country_user_policy_and_pagination_filters() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + + detail = client.get(f"/v2/user-policies/{ASSOCIATION_ID}?country_id=US") + page = client.get( + f"/v2/user-policies?country_id=us&user_id={USER_ID}" + f"&policy_id={POLICY_ID}&offset=2&limit=3" + ) + + assert detail.status_code == 200 + assert page.status_code == 200 + assert service.calls == [ + ( + "get_user_policy", + {"country_id": "us", "association_id": ASSOCIATION_ID}, + ), + ( + "list_user_policies", + { + "country_id": "us", + "user_id": USER_ID, + "policy_id": POLICY_ID, + "offset": 2, + "limit": 3, + }, + ), + ] + assert page.json()["result"] == { + "items": [detail.json()["result"]["item"]], + "offset": 2, + "limit": 3, + "has_more": False, + } + + +def test_collection_rejects_missing_unknown_duplicate_and_invalid_queries() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + + responses = [ + client.get("/v2/user-policies?country_id=us"), + client.get(f"/v2/user-policies?country_id=us&user_id={USER_ID}&search=reform"), + client.get(f"/v2/user-policies?country_id=us&country_id=uk&user_id={USER_ID}"), + client.get(f"/v2/user-policies?country_id=us&user_id={USER_ID}&limit=501"), + ] + + assert [response.status_code for response in responses] == [422] * 4 + assert all(response.json()["status"] == "error" for response in responses) + assert service.calls == [] + + +def test_patch_supports_explicit_null_and_rejects_identity_or_empty_changes() -> None: + service = FakeUserPolicyService() + client, flask_calls = _client(service) + + cleared = client.patch( + f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us", + json={"name": None}, + ) + identity = client.patch( + f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us", + json={"policy_id": str(uuid4())}, + ) + empty = client.patch( + f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us", + json={}, + ) + + assert cleared.status_code == 200 + assert cleared.json()["result"]["item"]["name"] is None + assert identity.status_code == 422 + assert empty.status_code == 422 + assert [call[0] for call in service.calls] == ["patch_user_policy"] + assert flask_calls["count"] == 0 + + +def test_delete_returns_no_content_and_uses_country_scoped_identity() -> None: + service = FakeUserPolicyService() + client, flask_calls = _client(service) + + response = client.delete(f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us") + + assert response.status_code == 204 + assert response.content == b"" + assert service.calls == [ + ( + "delete_user_policy", + {"country_id": "us", "association_id": ASSOCIATION_ID}, + ) + ] + assert flask_calls["count"] == 0 + + +@pytest.mark.parametrize( + ("error", "status", "message"), + [ + (AssociationCountryConflictError("different country"), 400, "country"), + (AssociationPolicyNotFoundError("policy was not found"), 404, "not found"), + (AssociationUserNotFoundError("user was not found"), 404, "not found"), + (UserPolicyNotFoundError("association was not found"), 404, "not found"), + (V2ConfigurationError("postgresql://secret"), 503, "unavailable"), + ( + OperationalError("statement secret", {}, Exception("credential")), + 503, + "unavailable", + ), + (TimeoutError("pool timeout"), 503, "unavailable"), + (SQLAlchemyError("statement secret"), 503, "unavailable"), + (RuntimeError("credential and caller data"), 500, "operation failed"), + ], +) +def test_association_failures_map_to_secret_safe_typed_errors( + error: Exception, + status: int, + message: str, +) -> None: + service = FakeUserPolicyService() + service.error = error + client, _flask_calls = _client(service) + + response = client.get(f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us") + + assert response.status_code == status + assert response.json()["status"] == "error" + assert message in response.json()["message"].lower() + assert "secret" not in response.text + assert "credential" not in response.text + + +def test_openapi_publishes_complete_no_auth_association_contracts() -> None: + service = FakeUserPolicyService() + client, _flask_calls = _client(service) + schema = client.get("/v2/openapi.json").json() + + assert { + path for path in schema["paths"] if path.startswith("/v2/user-policies") + } == { + "/v2/user-policies", + "/v2/user-policies/{association_id}", + } + collection = schema["paths"]["/v2/user-policies"]["get"] + parameters = {item["name"]: item for item in collection["parameters"]} + assert parameters["country_id"]["required"] is True + assert parameters["user_id"]["required"] is True + assert "does not prove caller control" in parameters["user_id"]["description"] + assert parameters["user_id"]["schema"]["format"] == "uuid" + assert parameters["limit"]["schema"]["maximum"] == 500 + assert set(schema["paths"]["/v2/user-policies"]) == {"get", "post"} + assert set(schema["paths"]["/v2/user-policies/{association_id}"]) == { + "get", + "patch", + "delete", + } + + for path, method in ( + ("/v2/user-policies", "post"), + ("/v2/user-policies", "get"), + ("/v2/user-policies/{association_id}", "get"), + ("/v2/user-policies/{association_id}", "patch"), + ("/v2/user-policies/{association_id}", "delete"), + ): + operation = schema["paths"][path][method] + assert "security" not in operation + assert {"400", "404", "409", "422", "500", "503"} <= set(operation["responses"]) + + item_schema = schema["components"]["schemas"]["UserPolicyItem"] + assert set(item_schema["required"]) == { + "id", + "country_id", + "user_id", + "policy_id", + "name", + "description", + "created_at", + "updated_at", + } + patch = schema["paths"]["/v2/user-policies/{association_id}"]["patch"] + patch_ref = patch["requestBody"]["content"]["application/json"]["schema"]["$ref"] + patch_schema = schema["components"]["schemas"][patch_ref.rsplit("/", 1)[-1]] + assert patch_schema["additionalProperties"] is False + assert set(patch_schema["properties"]) == {"name", "description"} + + +def test_native_association_routes_do_not_use_cloud_sql_or_flask( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_cloud_sql(): + raise AssertionError("Cloud SQL must not be selected") + + monkeypatch.setattr( + "policyengine_api.data.orm.get_v1_session_factory", + reject_cloud_sql, + ) + service = FakeUserPolicyService() + client, flask_calls = _client(service) + + response = client.get(f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us") + + assert response.status_code == 200 + assert service.calls[0][0] == "get_user_policy" + assert flask_calls["count"] == 0 diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py new file mode 100644 index 000000000..3444bf9ae --- /dev/null +++ b/tests/unit/v2/test_user_policy_service.py @@ -0,0 +1,234 @@ +"""Service tests for native v2 user-policy associations.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import UUID + +import pytest +import sqlalchemy as sa +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session, create_engine, select + +from policyengine_api.data.v2.models import ( + LegacyUserPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + User, + UserPolicy, + V2_METADATA, +) +from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, +) +from policyengine_api.services.v2.user_policies.services import V2UserPolicyService +from policyengine_api.services.v2.user_policies.types import ( + UserPolicyCreationInput, + UserPolicyUpdateInput, +) +from policyengine_api.services.v2.user_policies.validators import ( + AssociationCountryConflictError, + AssociationPolicyNotFoundError, + AssociationUserNotFoundError, + UserPolicyNotFoundError, +) + + +USER_ID = UUID("00000000-0000-0000-0000-000000000070") +OTHER_USER_ID = UUID("00000000-0000-0000-0000-000000000071") + + +@pytest.fixture +def association_store(): + engine = create_engine("sqlite://") + + @sa.event.listens_for(engine, "connect") + def enable_foreign_keys(dbapi_connection, _connection_record) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + V2_METADATA.create_all(engine) + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + with sessions.begin() as session: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version="5.2.0", + current_law_id=1, + metadata_time_periods=[2026], + ) + parameter = Parameter( + name="gov.example.rate", + tax_benefit_model_version=version, + ) + policy = Policy( + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="a" * 64, + ) + value = ParameterValue( + parameter=parameter, + policy=policy, + value_json=0.2, + start_date=datetime(2026, 1, 1, tzinfo=timezone.utc), + ) + session.add_all( + [ + value, + User(id=USER_ID, primary_country="us"), + User(id=OTHER_USER_ID, primary_country="us"), + ] + ) + session.flush() + identity = (policy.id, value.id) + + yield V2UserPolicyService(UserPolicyDatabaseSession(sessions)), sessions, identity + engine.dispose() + + +def _command(policy_id, **changes) -> UserPolicyCreationInput: + values = { + "country_id": "us", + "user_id": USER_ID, + "policy_id": policy_id, + "name": "Saved reform", + "description": "Personal note", + } + values.update(changes) + return UserPolicyCreationInput.model_validate(values) + + +def test_create_allows_distinct_duplicate_links_for_an_existing_user( + association_store, +) -> None: + service, sessions, (policy_id, _value_id) = association_store + + first = service.create_user_policy(_command(policy_id)) + second = service.create_user_policy(_command(policy_id, name="Second save")) + + assert first.id != second.id + assert first.user_id == USER_ID + assert second.policy_id == policy_id + with sessions() as session: + assert len(session.exec(select(UserPolicy)).all()) == 2 + + +def test_create_rejects_missing_policy_and_country_conflict( + association_store, +) -> None: + service, _sessions, (policy_id, _value_id) = association_store + + with pytest.raises(AssociationPolicyNotFoundError): + service.create_user_policy(_command("00000000-0000-0000-0000-000000000099")) + with pytest.raises(AssociationUserNotFoundError): + service.create_user_policy( + _command( + policy_id, + user_id="00000000-0000-0000-0000-000000000099", + ) + ) + with pytest.raises(AssociationCountryConflictError): + service.create_user_policy(_command(policy_id, country_id="uk")) + + +def test_detail_list_filter_and_pagination_are_country_scoped( + association_store, +) -> None: + service, _sessions, (policy_id, _value_id) = association_store + first = service.create_user_policy(_command(policy_id, name="First")) + service.create_user_policy(_command(policy_id, name="Second")) + service.create_user_policy(_command(policy_id, user_id=OTHER_USER_ID, name="Other")) + + detail = service.get_user_policy( + country_id="us", + association_id=first.id, + ) + page = service.list_user_policies( + country_id="us", + user_id=USER_ID, + policy_id=policy_id, + limit=1, + ) + second_page = service.list_user_policies( + country_id="us", + user_id=USER_ID, + offset=1, + limit=1, + ) + + assert detail.id == first.id + assert [item.name for item in page.items] == ["First"] + assert page.has_more is True + assert [item.name for item in second_page.items] == ["Second"] + with pytest.raises(UserPolicyNotFoundError): + service.get_user_policy(country_id="uk", association_id=first.id) + + +def test_patch_changes_only_supplied_fields_and_supports_null_clearing( + association_store, +) -> None: + service, _sessions, (policy_id, _value_id) = association_store + created = service.create_user_policy(_command(policy_id)) + + renamed = service.patch_user_policy( + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput(name="Renamed"), + ) + cleared = service.patch_user_policy( + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput(description=None), + ) + + assert renamed.name == "Renamed" + assert renamed.description == "Personal note" + assert cleared.name == "Renamed" + assert cleared.description is None + assert cleared.updated_at >= created.updated_at + assert (cleared.country_id, cleared.user_id, cleared.policy_id) == ( + "us", + USER_ID, + policy_id, + ) + + +def test_delete_removes_mapping_but_preserves_policy_and_parameter_value( + association_store, +) -> None: + service, sessions, (policy_id, value_id) = association_store + created = service.create_user_policy(_command(policy_id)) + with sessions.begin() as session: + mapping = LegacyUserPolicyMapping( + country_id="us", + legacy_user_policy_id=42, + user_policy_id=created.id, + fingerprint_version=1, + fingerprint_sha256="b" * 64, + ) + session.add(mapping) + session.flush() + mapping_id = mapping.id + + service.delete_user_policy(country_id="us", association_id=created.id) + + with sessions() as session: + assert session.get(UserPolicy, created.id) is None + assert session.get(LegacyUserPolicyMapping, mapping_id) is None + assert session.get(Policy, policy_id) is not None + assert session.get(ParameterValue, value_id) is not None + + +def test_patch_command_rejects_empty_and_immutable_fields() -> None: + with pytest.raises(ValueError): + UserPolicyUpdateInput() + with pytest.raises(ValueError): + UserPolicyUpdateInput.model_validate( + {"policy_id": "00000000-0000-0000-0000-000000000001"} + ) diff --git a/uv.lock b/uv.lock index de2dde78d..49e9c2757 100644 --- a/uv.lock +++ b/uv.lock @@ -2,9 +2,12 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -1698,6 +1701,122 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "librt" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/9b/356320fbae2ac8467e21c5e73e1389c80468e4998c62cc7d3536cc51b614/librt-0.15.0.tar.gz", hash = "sha256:4e66cbe84437497d951b799d3e1551291b6fb3d643820a7014b3655d57a59162", size = 214338, upload-time = "2026-08-07T10:49:42.663Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/52/06790ced2ac7117f890c21bda43c39c958ec82aa665c0718e821d33ff939/librt-0.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:823b92cf3c18ecd08afc70c42473888b41b6e8ef5046f3b82c05c154a2fa3d22", size = 148039, upload-time = "2026-08-07T10:46:41.165Z" }, + { url = "https://files.pythonhosted.org/packages/e7/1d/8e150b7fc449a1f33c8a760965cc1f43b14fc1577d9d0b50ab2701420e74/librt-0.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c70bc1b602cf59917e8f0c7a2cbc8bcc6fbc14d5486136b00707a79619121d63", size = 153067, upload-time = "2026-08-07T10:46:42.418Z" }, + { url = "https://files.pythonhosted.org/packages/51/87/a162bc5a66a35599dc619ecb215145f4de7d68e886b479b6d12593139f7c/librt-0.15.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:814ff83a25b5fce8b9c80c4dd803153fb5c5599fc74db9e022466938368957ef", size = 493087, upload-time = "2026-08-07T10:46:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/e5/3a/aeea1fc620cf48060d3065b37614edbf97043c099d0f50782bc8ca61d897/librt-0.15.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:57f5eeb6ad4c180de583b1038e61fe5fbd9796bb69a8a1c1a0c7ddbec4c8c60f", size = 485608, upload-time = "2026-08-07T10:46:45.038Z" }, + { url = "https://files.pythonhosted.org/packages/52/ff/fe571ad416f0856fd0d5578ffc2e6dc531891e586e36b647bcf50569cab8/librt-0.15.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82909c8f7eb9952656b65d3147afde4cf8e6d5a991eebc86418b5e65843b0ab8", size = 498723, upload-time = "2026-08-07T10:46:46.35Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e1/7a65eb5dedb1f00aebd948cdd8e17add48bf066cab3514e9daf84ab45a6c/librt-0.15.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f779070399f991400fc451719e0ea388eb7de313388bada2c127a35de05f798a", size = 516002, upload-time = "2026-08-07T10:46:47.599Z" }, + { url = "https://files.pythonhosted.org/packages/5f/45/59832b0ebfbd08c2742e6ece372ceb53f18bf1faef5d33c8daf3abebf749/librt-0.15.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bac89069bc496ebdf4f79ebb57bbd10d0b214c8454225deb672d91002bd17e18", size = 508607, upload-time = "2026-08-07T10:46:48.873Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0d/37fa73f3b43ebd8259f91ae9102a15e5a54e65d581e48dea72df3e81d7a4/librt-0.15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e0d00c708fb2f5822b152429b1ac80a58dbbbc3f6c232c4d13a3f7fcf2ea5b4c", size = 530422, upload-time = "2026-08-07T10:46:50.45Z" }, + { url = "https://files.pythonhosted.org/packages/26/02/e046c6fe7a5881ac34623242192f484426ba8a75595fd18f22c53a3f530f/librt-0.15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6c6624fe268625869485553dd7cc1daf30d22558215bb2a4ff16f67a9801a31a", size = 534303, upload-time = "2026-08-07T10:46:51.693Z" }, + { url = "https://files.pythonhosted.org/packages/95/32/d5e6d861ab0366f3edf74f887ab0c9eb9f535aaf01d32b80b4f734daa179/librt-0.15.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f56b397858a23dacf35ede366ed2212fdc03a6a57a1ad36468ad6e9dc5fac091", size = 536084, upload-time = "2026-08-07T10:46:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/2a/de/d69d725513fe53fc90c6d7a1f86e4428939bad2fb905b17fe4c18d413dde/librt-0.15.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4388184646efe2054911c5b00a1077d6d1ee86a95b7e8ba96dc7850a809f3f40", size = 514307, upload-time = "2026-08-07T10:46:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/36/93/f8aded0d6682b4f25820fa86e0690f87f01df9fd7bd09ddb04d9167ad021/librt-0.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:97335f59082f9fe2ce6c2a9cc6433a0114bbb6cd4d5c09dd76c95c68b9f9a8b0", size = 557686, upload-time = "2026-08-07T10:46:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/74/09/ffeb6bdeb6cd862b4272fddc8ad05f938dd25d020ed517e631813917d80a/librt-0.15.0-cp311-cp311-win32.whl", hash = "sha256:83380ffde38062a2e9bb55d83e74474f6614665528b98a6928720fc006dfffbb", size = 104917, upload-time = "2026-08-07T10:46:56.605Z" }, + { url = "https://files.pythonhosted.org/packages/96/28/7e2313a3ffbf0b4de7ba3da58a09e488507b4bd1ea2b5e69378354a23415/librt-0.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:f75720477ee05d509a310e856cacc8d909adc182f7b91193c207bcc26d7ee6db", size = 125886, upload-time = "2026-08-07T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/39/9e/04b8c3cde014ef255ee785730425268354543acc38902093a40afa0dc164/librt-0.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:256237037a3ab001ae8d9803b2d43562a4c3aa38739843694349e4d5ebb0fd56", size = 111885, upload-time = "2026-08-07T10:46:58.787Z" }, + { url = "https://files.pythonhosted.org/packages/ba/39/99c25030e782bdfb7a21be8c05254806a2e4bbb05c8d50c2a2130acbfa05/librt-0.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e87bc679f86a99aa3b26e3c78eeb821a247c9a28eae48eaafcc32c3bf4c3bb9e", size = 151021, upload-time = "2026-08-07T10:47:00.057Z" }, + { url = "https://files.pythonhosted.org/packages/14/43/f4b1bd1b2888798a1409808889a25ea1ba49eaabce7d681ed27734c2df9d/librt-0.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:71599e011ac880e8e45d46047d714871894c7d4ab6f25626f8d4f89da21f368d", size = 155267, upload-time = "2026-08-07T10:47:01.311Z" }, + { url = "https://files.pythonhosted.org/packages/0c/db/3ad9c965c72f1e1d6beeec44ec10a54e17be8ae042fbb4baade16cbadced/librt-0.15.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c802434092b769b1d613ed2e13fac15fbfce1934a74bd10283b03c0fae231cd1", size = 503136, upload-time = "2026-08-07T10:47:02.45Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/5888a6d76acd62ebce66c61b74d94e9370b9c32929f111e487bb6546f8ed/librt-0.15.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5500eeae393a184d14e1f35645962c27129d20c81afa4069e6ef826ebc2b3aaa", size = 496670, upload-time = "2026-08-07T10:47:03.675Z" }, + { url = "https://files.pythonhosted.org/packages/29/39/ab57cc2f5b276156da02bb7f5a8921bada1cb1993ffec99acf811c602c23/librt-0.15.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6ecfc32dfb46fb7b565bcd6abf9412acf978775a998273d22888a6d7953730dd", size = 513688, upload-time = "2026-08-07T10:47:04.981Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b9/bdbb0b648b5c2befb031f4c6f3b1dd857415e8fb492a25a3c764a6681e6c/librt-0.15.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cc46cfd15022e35084355478c9ac809d90b1152222706ac9a7655ec21df6fa", size = 531904, upload-time = "2026-08-07T10:47:06.211Z" }, + { url = "https://files.pythonhosted.org/packages/93/26/473c2e4b6c104e9e58e27ce95fc8005c8bd4fc36cae4f254371125a92db8/librt-0.15.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5f51401d102c885b9ca509e62c79b1dbff286e1b9b047fde6f763780789356d", size = 524427, upload-time = "2026-08-07T10:47:07.592Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/03b3abb82b41714671b907bf6989b228e31e6a8af52dec82b5b0728dc250/librt-0.15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cc30523e3f1a23fb7511cc659834a0d01a1042bb9de359bc1c131cc4ec6c9656", size = 543155, upload-time = "2026-08-07T10:47:08.866Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0e/9bb1f0a4affbd0a1888f4f79dc03ed2a299d9a2c26c59ab2a97dcbf11903/librt-0.15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:59fe030d8ae4a57e3fb7756bf35a858de74e04066fc8555c53d0af979132af81", size = 546890, upload-time = "2026-08-07T10:47:10.327Z" }, + { url = "https://files.pythonhosted.org/packages/dc/84/6937a280d461f7de6e031ffb02edc2b7c3c90d49d630565ce8ff27cbc5f2/librt-0.15.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5a6526a2a956bbb1e4ae3568c82e650fc99119c66bb011ea60715744955a2b4d", size = 555163, upload-time = "2026-08-07T10:47:11.798Z" }, + { url = "https://files.pythonhosted.org/packages/bc/95/2a2853c1ee014bf102116e7f897a04beeaeb2461b45b79af98bdfb95f1ef/librt-0.15.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:85ea21ec6730194d67156b0e0b5430ccb1d61f8b8b907e39b37f9812b74a13f0", size = 535812, upload-time = "2026-08-07T10:47:13.279Z" }, + { url = "https://files.pythonhosted.org/packages/c9/4c/cf9601c1b4c5f09280acd5d83abdb2e68527a2be8257136eb42304218622/librt-0.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1e47b8ba865d7ede071a91a7163073bbaeb72541f1ef8a07d512c45c7b5007f2", size = 573688, upload-time = "2026-08-07T10:47:14.727Z" }, + { url = "https://files.pythonhosted.org/packages/47/6d/9ac7cbec46189a7625af4b5acbd25f10d827f4141b2002181848c8418923/librt-0.15.0-cp312-cp312-win32.whl", hash = "sha256:a5207ec414d1c4a2a7231b2086970dc036f94293cdf338190984958a013a42f1", size = 106138, upload-time = "2026-08-07T10:47:15.973Z" }, + { url = "https://files.pythonhosted.org/packages/38/d0/2ae99c83be86ce23f925ac1aeeedc777e97f427c4a8d190c70d0a16e9a87/librt-0.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:73b30cfa976659b3917c8f6153bdb0591c6a9ec6583599fd24a689b690622022", size = 126974, upload-time = "2026-08-07T10:47:17.049Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ef/dd24f9635c730b86b87587967dda7516b1845e8b17684603d31607fed598/librt-0.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:a54cf9e0ef47b96af580849db5471142200568ce1e02cbf416addab551369570", size = 112292, upload-time = "2026-08-07T10:47:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/e7/42/467b53a601b406ccd7b97c1fd54b59cb34f9185ad5ce7e9d5c3c4e8961c8/librt-0.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:db13ca398005abcbe538deda87b686d9bd08b7001cf40c4c06b444960ae10a26", size = 151029, upload-time = "2026-08-07T10:47:19.312Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/36c2299b7a94b84fdd01220d8a777a71be5be0925bb0dbdf71c0a06a34d9/librt-0.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa1f1995789dca3698bc550aaceb09a51bd5df0a057ff84ff15296cd1975b801", size = 155194, upload-time = "2026-08-07T10:47:20.398Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/ed5071f9325845e670bd36012757419767fbf56af77ed483077b9e4db541/librt-0.15.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55456ea87d8df21808446d03817be2f65e20391c1c615d9187440dff28cd08dc", size = 502568, upload-time = "2026-08-07T10:47:21.652Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/6450c67c3615d87704bcbc21323fafc69c799b06a044c447529f725d4b01/librt-0.15.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5a86a5a08c2235316bdb359d5dbb6ce0abfca7fac06363103e2c5af571d92f95", size = 496153, upload-time = "2026-08-07T10:47:22.925Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d6/5f52b722bc75076954b3bfd49be15ea362df4d580c6fb315d0f617100d30/librt-0.15.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e56b6a368529bed262da40ce13f8fef590db0479819cca84f16a1f01ac356d0b", size = 513336, upload-time = "2026-08-07T10:47:24.213Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e2/c08fd1d36ce63ea5a12b85c5d37f4550b5f86a692167e41e5a74222607ae/librt-0.15.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:234d8d394721fa0d786af15ebf1f3fb7f3ed82fd1cd0cde45c2f247b5d4281d2", size = 531661, upload-time = "2026-08-07T10:47:25.507Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d8/d9482fcbeb177b9eb87bb3899eeb3b42be690313c652f9e146b1d0681fb2/librt-0.15.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d8363d7accb0286ac3a0e633f396e93800dafb8150494505daf9515bbda591f3", size = 524487, upload-time = "2026-08-07T10:47:26.79Z" }, + { url = "https://files.pythonhosted.org/packages/10/cc/075171517b41f861753034fbb151b42cfc83bcc853849f24f5e66fd60ccf/librt-0.15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0f0ee3644d951f31055ad07d77d92520e84505dd7a432cc4cd501dd70ee06785", size = 543201, upload-time = "2026-08-07T10:47:27.999Z" }, + { url = "https://files.pythonhosted.org/packages/b0/03/42c2330f37eeb475b6affeedd06518f60035f323af3a839335e3fc9fef2d/librt-0.15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2cfd1a81a648806e6a7717be4cc4d1bb392fa229752bf8444ba365e381e984d6", size = 546467, upload-time = "2026-08-07T10:47:29.396Z" }, + { url = "https://files.pythonhosted.org/packages/57/1e/1ad4c5638f7e64d8560328bd25c54b409a661bdb6ff254b38ff90744288d/librt-0.15.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a6cd22c9da0d866558e46a041f1cc0c2bbb26b61b137b2347fa834c332e1d101", size = 555139, upload-time = "2026-08-07T10:47:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/49/41/39fa7d15db1204cd1cbe6514680fbdc243adf754a0885061308f43afc013/librt-0.15.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:6d5225ef8801e4ea5e482fa9b5dfb891dd9ef6f6d870f1f25d449ca2c70ac218", size = 536050, upload-time = "2026-08-07T10:47:32.222Z" }, + { url = "https://files.pythonhosted.org/packages/1e/88/c6dcf0dd8e26dc0c9a499a2abab8646c86dcaf9ecea9524cb46d3686331a/librt-0.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d28a05796b99f749bf8794f17ba9ba1612d0076b802e9cfc62c554634e9ce3b", size = 573700, upload-time = "2026-08-07T10:47:33.527Z" }, + { url = "https://files.pythonhosted.org/packages/1b/9b/ab54c71a7918a7c34fa5327fb61390a77446a07a146fbfb1165250a61035/librt-0.15.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2067ff438048cead9d223ca5675bae2a25e520a7c3e6c1498bf9c6892d22caab", size = 82194, upload-time = "2026-08-07T10:47:34.835Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b2/4f9a243bb892395f3becb80789ade13771701091f9f07ab8230247953ba8/librt-0.15.0-cp313-cp313-win32.whl", hash = "sha256:1cd3b721f24c206398b9e26da3c3a9c011e6e89d06f318ba8ebefc30f1003890", size = 106231, upload-time = "2026-08-07T10:47:36.251Z" }, + { url = "https://files.pythonhosted.org/packages/bf/af/64aff4885a40b93132382f2c314647d722574605416504379184ef3045ea/librt-0.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:f395a4a9a03ac062dbe9a9f82e0c720502e590a38feee6a757bc82e9c63afbd8", size = 126996, upload-time = "2026-08-07T10:47:37.453Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/335bccf6c7cb9028cb0b54aead27d9ece3f01f83bc6baa2abace5da655c1/librt-0.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:0a15cb554761247d84a3ec0cbdf4078d70725384f0e4662c0fa3b26266eb60ad", size = 112188, upload-time = "2026-08-07T10:47:38.729Z" }, + { url = "https://files.pythonhosted.org/packages/a8/93/949053fb462eecc4a9a5ee770a81f4b40be7b79538b245545d4aebc6b58b/librt-0.15.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f5de7feedc56337a088eb15cd9fafa9938367362221d8cc62c642b7f94821993", size = 149833, upload-time = "2026-08-07T10:47:39.86Z" }, + { url = "https://files.pythonhosted.org/packages/61/ca/8281aa6cd560a3420e4497729f6b704b53be3eeaaef82d5aeadddaf7441f/librt-0.15.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c0eb900c0e91f4aebe680845242e614f1864edfd44106380d0752ac29522bf8", size = 154088, upload-time = "2026-08-07T10:47:41.065Z" }, + { url = "https://files.pythonhosted.org/packages/dd/02/1a1662dceaba6a086360891448d5ce9a7d3555976cae59a31a39d744b9c7/librt-0.15.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8c9a650a188e38bac005048cbe6342e81407782944d01934540ab75e417df21", size = 494215, upload-time = "2026-08-07T10:47:42.388Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/99211619dc656370a3740c33d2b0b6d5a3fb1e73689314f6ed477a397dc4/librt-0.15.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:92bfed8deec93df30286b9fe9e3b1dd17329cc076a192b4ee5ec223841d54953", size = 491173, upload-time = "2026-08-07T10:47:43.683Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/5448d0b05f4579b635d3899176817ebf561af0e57bacd425b5b1887264c1/librt-0.15.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec4b19788f835711a2072f9dbe6b03b3bf32ed1f0fb30cf399bdd59d9f0c33fa", size = 505512, upload-time = "2026-08-07T10:47:45.314Z" }, + { url = "https://files.pythonhosted.org/packages/95/82/01940e40b83c43a546c4a3c896cf34ca272a9690899d55914e4827b3dcce/librt-0.15.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4c7bacb70930f3d0a56f4ecf1be474a1f0d941b01dd73b756f3c256d42cb879", size = 523073, upload-time = "2026-08-07T10:47:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/88/fa/759c0030f3ee371439eb26de34fc745807caf0abb878af7af4b8b7c3dd3d/librt-0.15.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3e79f05e4a08b4d880342673312bbc895b56df7765605796f15902eb5367d3ae", size = 515080, upload-time = "2026-08-07T10:47:48.319Z" }, + { url = "https://files.pythonhosted.org/packages/0b/27/894e072228fcb159703c655da69f8cd10dbed489c36e3df7dd032a2483be/librt-0.15.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a417149c0cba4d50b61e992e5a15e69eaf96746609b461cc4ed168aeef6b79dd", size = 534164, upload-time = "2026-08-07T10:47:49.875Z" }, + { url = "https://files.pythonhosted.org/packages/98/a3/0078e91c1f36f8815db17827de15650b9a3fe56c55fbf998c854b34e40d3/librt-0.15.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:da7a94d6a3411f579d72aa3e3bc5fbca7ed4549f3dbd7e5de3aa567333374285", size = 540616, upload-time = "2026-08-07T10:47:51.408Z" }, + { url = "https://files.pythonhosted.org/packages/86/33/81a29b796dd52a45e9ef7974c7732926e8f10f15b8d2be505665979f896d/librt-0.15.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:856f743ae607f2c1380eccb566c0038a9fb3eabf0fc2be2704d76d9f73557239", size = 545890, upload-time = "2026-08-07T10:47:52.818Z" }, + { url = "https://files.pythonhosted.org/packages/05/82/8be1baa1350e5d30cfd70ae79d0a6f4dc5862ef47f7bb2808aabc9bb86e5/librt-0.15.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:779a6e7c894737e5983e7790a9c78c4000c30e23c9aada08081bdbea53b0fa60", size = 523287, upload-time = "2026-08-07T10:47:54.165Z" }, + { url = "https://files.pythonhosted.org/packages/c6/4f/d1be6a01a35c20ef734e0e44113f87d4af756a9354a89dcfbe3b4f8af5e1/librt-0.15.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:96bb17dbe8bab3c0954fbebfc69ed395599de75b6bbc35e3270a878e15d4dd65", size = 565868, upload-time = "2026-08-07T10:47:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/67/88/649cfa33f5825927b160610f670bdab012a64d627eddb94fa795ea4292fd/librt-0.15.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:7220697efaa6e5348fc3d18ee7f8563d4bfecd9872b37ffb915bfc1d08840622", size = 81619, upload-time = "2026-08-07T10:47:56.886Z" }, + { url = "https://files.pythonhosted.org/packages/22/31/8e88a8d5e48fc8d1a817787fb6811dfff6499acd6c8683dd83934aa6ede0/librt-0.15.0-cp314-cp314-win32.whl", hash = "sha256:f54598964d357b1c5ab77cf5d92f21e598fe0e23cdbe9618480807f81b4eba15", size = 100138, upload-time = "2026-08-07T10:47:58.093Z" }, + { url = "https://files.pythonhosted.org/packages/80/92/20fd6c4b6a1b1a564b076d55cd3d427d8428217d7638dc25a654cc4791d4/librt-0.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:3ff5893a2c23d886aa9ce786de5ac6ddc74aeeaf90743682b74d920e117d2e28", size = 121258, upload-time = "2026-08-07T10:47:59.564Z" }, + { url = "https://files.pythonhosted.org/packages/fc/28/6af430b44d9ebb897b865a3c363b6dcace51357be2347cc0f8f869656a86/librt-0.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:3722a099730704c9a3d70c879fc0f51daec25fe5f1555672d97bc595abeafb95", size = 106467, upload-time = "2026-08-07T10:48:01.097Z" }, + { url = "https://files.pythonhosted.org/packages/7e/aa/b42bb798942ced219f6d63b27e07f91237887a8d0bd0921666db79a13790/librt-0.15.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:38c0c7d4b6fc06c3324b3f9162c8391bfc4fd9dde53afe1033ce7edb48d5a714", size = 159523, upload-time = "2026-08-07T10:48:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/75/03/1b53cd4ef904e73b1d828a5f90143bf94a2967d7cfff0b9ccf93e12aa9b4/librt-0.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8b2fdd7ead3c995c37940a790690660d0ca006c302db26cc51933f6766866fc3", size = 161638, upload-time = "2026-08-07T10:48:03.725Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c4/9f9c9fba097d49e9e694c2b4dc331df31884645ecbc58a93b4b5fc69d2c5/librt-0.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2fde98cf1fc4bac144ce23c2c4c017b924ba714509ea9334977b0b27050c837d", size = 701795, upload-time = "2026-08-07T10:48:05.135Z" }, + { url = "https://files.pythonhosted.org/packages/4c/05/0966840bda0380c8ae167b9043c6230202941cc90ea29c48e096964c765e/librt-0.15.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:e3b461183c5fa7681b48560f91515f53a953122fb30c71e07abc67d7ddf58c38", size = 682147, upload-time = "2026-08-07T10:48:06.555Z" }, + { url = "https://files.pythonhosted.org/packages/18/af/1c47ca573c30ea47d195aec26133af522fea1104afaace028d7b32247ea8/librt-0.15.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4bbcc257e3babea20a91715c361b24554ec4e8f51aa578568afc230799fe1a19", size = 696397, upload-time = "2026-08-07T10:48:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0f/1aed6223d4f9f9d1171a8596ff100ea4c3f7699fea7a4ba657c3e60daa6c/librt-0.15.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b845b8d48088fad0cadc84be4b8fda63203be7e9237b71015b3925443c1f35ab", size = 722542, upload-time = "2026-08-07T10:48:09.569Z" }, + { url = "https://files.pythonhosted.org/packages/c6/22/9e3a929aea456c97d69e6ef3884efea56d4807f97399471cc946baebd8af/librt-0.15.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b30e600e8f337b9bd7f39b86d9fdfedc73cc46e3d0f745931a23a234220bb7e2", size = 729709, upload-time = "2026-08-07T10:48:11.129Z" }, + { url = "https://files.pythonhosted.org/packages/e9/1b/c327ef6018e3a9ca0b8e7c5eddeeb331ba8f9b76c24e126d37d0f6d62faf/librt-0.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:64b0c8c35aa4c4ed79896359f3e0b285cbe4e610042106500da4811c322cc108", size = 752891, upload-time = "2026-08-07T10:48:12.558Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d1/d5f1ea02c56930087009e39db9b70660a663e76c730b27b925d786718457/librt-0.15.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0da0d94cb802f32a0524653e7201f2cef72d5f700a5407678f5290483d4fcd08", size = 745301, upload-time = "2026-08-07T10:48:14.55Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3c/5f7c585d15ebb2250c73e7c0ee4e9e47be72c65d520c07ddbcdc62037674/librt-0.15.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4a6369168d371207339b1e50d4532b06a7121586141f82599505a3f315751d47", size = 747921, upload-time = "2026-08-07T10:48:16.453Z" }, + { url = "https://files.pythonhosted.org/packages/7f/52/1443a446486eba966bcbca1696b472e4f210320ec42f490a47f48fbf0fdc/librt-0.15.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c434e072557ade9cbc642d052c89d031efe47d5c9614523619d0d74a02378e81", size = 727561, upload-time = "2026-08-07T10:48:18.089Z" }, + { url = "https://files.pythonhosted.org/packages/79/91/2270a9380f11725cf83ce1925a5e32dd1dde2be9bba597f25c10a38644e7/librt-0.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7eec6a42018bc1d45763b1c162d3d2bf7c3b9a1b0ed30d3e91dcba390efefcc", size = 774417, upload-time = "2026-08-07T10:48:19.611Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/f4b1548d4f5b99186737fe27aec238e9823e8d5d23bf4df007c030689dc5/librt-0.15.0-cp314-cp314t-win32.whl", hash = "sha256:6912fa5e635d74529ac7cdb1bdf6ca3af4453da8d1edbe0110ee1cb4ad407ebf", size = 104381, upload-time = "2026-08-07T10:48:21.048Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/134afad262def1de04c0843c376d02135f1168af43f22e09a52bd8394727/librt-0.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8e11699ed745931c395acd3621b07062e0f840efa6935aad87a64ed0995f0915", size = 127034, upload-time = "2026-08-07T10:48:22.561Z" }, + { url = "https://files.pythonhosted.org/packages/99/5f/1b6846b20572bd699c9e9ec321a5f781845bee477df2aa2a43b28bc40119/librt-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5d2a91724463bfed4f573cd7a9fdc856d2e230d0c0e5a61416a93481dccd8605", size = 110827, upload-time = "2026-08-07T10:48:23.804Z" }, + { url = "https://files.pythonhosted.org/packages/c6/44/4de9f4ddadb009a55c7758eb5736d62534a7daaf27bd71bc50e64b606b06/librt-0.15.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:8443e38dcfcfdbcf5add5118c623efd788d65ac2e25756d6251a54a06a4d0aca", size = 149843, upload-time = "2026-08-07T10:48:25.148Z" }, + { url = "https://files.pythonhosted.org/packages/1f/eb/5d9ab71e30119c44094e0275f38b47dd327aea0f843a080396677029d508/librt-0.15.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6d15a29033c57490cfe2069097c6fc4049e4e65ffbb749be7dc453b7c4c68965", size = 154510, upload-time = "2026-08-07T10:48:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/8505d1b8f5e8c19587bd03f7429993b3e9ce5c06819d856bfb11d919374c/librt-0.15.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2c05c729b589e734c09578bf5964be48a911765484840d017bbc84f49d4c4ad", size = 497543, upload-time = "2026-08-07T10:48:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9a/3a8390775cb095765aded027ac9c63e7c8ea74e731498607544c6505de0e/librt-0.15.0-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:fa60887537e1d0cd2d9982269d33a709bf54b195cd2b9364fc0a758022af5bd9", size = 480452, upload-time = "2026-08-07T10:48:29.531Z" }, + { url = "https://files.pythonhosted.org/packages/e7/40/258a4a7117ee915d66de5cd9b8ade65a440993161107ce3a686f1859955c/librt-0.15.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d8bc24219b24c0af375718942ab75e3544b2763085f40f965be4326734ae8328", size = 507768, upload-time = "2026-08-07T10:48:31.007Z" }, + { url = "https://files.pythonhosted.org/packages/6b/c6/2f4dd296c97a0b85b98894519b279408ec9dd602d4f692b1ea0e25dee670/librt-0.15.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86a21a7bd3fe3a419512ef424cc1c020f6771d0b29cfddff36d1635a855e63f0", size = 525122, upload-time = "2026-08-07T10:48:32.7Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/29eab42be13b2bf0ea8cb227135a45d44693e30a7e8b92871981ff56b82b/librt-0.15.0-cp315-cp315-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dbab647e88d90b3167b91efe7091e248653688ed4337e4f90907a722c7361bb9", size = 520371, upload-time = "2026-08-07T10:48:34.294Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/4bad71adeca8fe208b775c2a35417fa5a2584c8f4791daaf89a89450fea1/librt-0.15.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d8edcf6f550e918dca779c069b9e156385c60b406f99fc7641f32c52f7193659", size = 537258, upload-time = "2026-08-07T10:48:35.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/63/59dba6143fdcc7240c54458b629f3250000a61b8945890fc9efd451b19c5/librt-0.15.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:8b62076030baa2d8b1501a46bf0e19c27a489aa90671c55665bff7887f7660b0", size = 527432, upload-time = "2026-08-07T10:48:37.466Z" }, + { url = "https://files.pythonhosted.org/packages/ec/21/21a24c6a2327d8362580efebe77286bf47b0f4062ec5ea41766e609d3c7d/librt-0.15.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:d00d20d1818e82a07a0ee0aa89a98b17ed7916b92441090b683719cb20a59b6d", size = 548108, upload-time = "2026-08-07T10:48:39.384Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6d/fc68c89a7971418b41f9a873623ff935cb864097544c6a2f8ce491c8ef5d/librt-0.15.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4e6ee93fc3cf848dcbf0cce2eca73d8e7dcd0cc2b6df3a529d57750b30a4c55c", size = 529681, upload-time = "2026-08-07T10:48:41.392Z" }, + { url = "https://files.pythonhosted.org/packages/65/7e/c2d98766124400d722063a630b0fde38a9fc768705d37eecca15c47dc192/librt-0.15.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:32896a0af72508ea979e0acb4e4c04cbeeae04938167950d535c83c45597167d", size = 567736, upload-time = "2026-08-07T10:48:43.124Z" }, + { url = "https://files.pythonhosted.org/packages/55/6c/f8c34a95e3a515c6e1c192b89511e7253c89a7760c6b500d57ffdb8d2dc8/librt-0.15.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:ec3ba415afaf951f6951b1dd16d3c8e4f540065fc382d7e70b823a79567ca374", size = 81673, upload-time = "2026-08-07T10:48:44.645Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/e23fa8e78679ec45728188650b39e8ff476c83b691c96f749217df3b1b7c/librt-0.15.0-cp315-cp315-win32.whl", hash = "sha256:d2813ba2503764f0450680c533d13df7cff9b49df1411062eded5f67db4195b9", size = 100081, upload-time = "2026-08-07T10:48:46.171Z" }, + { url = "https://files.pythonhosted.org/packages/e1/dc/3eb4c5e297343f0620a55532cd7c8d764d3001fa2159212dadf480464827/librt-0.15.0-cp315-cp315-win_amd64.whl", hash = "sha256:b87d67e33afaf265262f2a66db578284b88ee2e6fcd224579cb5c15518677ad8", size = 121228, upload-time = "2026-08-07T10:48:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/97/70/43abce19f04e49762f8ec834c8fafee13cc40fd6b94a72a24e534febfcd0/librt-0.15.0-cp315-cp315-win_arm64.whl", hash = "sha256:713bd7df21170b982e729e46870f31d6b437bd1a9b4648cffb529bd3c2ec5c4b", size = 106487, upload-time = "2026-08-07T10:48:49.095Z" }, + { url = "https://files.pythonhosted.org/packages/de/15/83f2deddb9368b8951ec8c9477269b5b9b8bd9bbf15e57402d0f38817dca/librt-0.15.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:3de789c82752730f94782a5ee518baf9c05edf85733aeaf73bb6e518755cdf54", size = 159448, upload-time = "2026-08-07T10:48:50.649Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/043097353f9b3c73b583d07f6b8e552795463f4bfc8caf85e42eee50c26a/librt-0.15.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:e0b5deec9a8664eb722c797241970fd4aa1894d25fda36a1ddac0f7407606bd6", size = 161686, upload-time = "2026-08-07T10:48:52.174Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2a/8ae77f9719d42ce71cd708560a3557b38ac3c17a0383e57f87084de45bbe/librt-0.15.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5563302a8359bc2295bb7084d1a8ed1519df96afb30eb2aa4e0bff7b54228988", size = 710668, upload-time = "2026-08-07T10:48:53.782Z" }, + { url = "https://files.pythonhosted.org/packages/61/34/c0436ea134deb9a0d6da80a396a2739a81cb31e0418f7227239e23140898/librt-0.15.0-cp315-cp315t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:22d6263b9d39d7bbb286fa791945646e3218f1be2d693e36fb630f1d0e59cd13", size = 679396, upload-time = "2026-08-07T10:48:55.645Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/001e0d99aa9250d5cd5715a9081291a20656083459f9019cda15255329e1/librt-0.15.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:39ffd14646190c454f0d86e0d256b33f00a87a26ab410e619773b841d0e41416", size = 704313, upload-time = "2026-08-07T10:48:57.46Z" }, + { url = "https://files.pythonhosted.org/packages/2d/53/b34fa9d0ff00f136f4d58ebb4c411ff634baed1eb412bb602a2bc8dcafcb/librt-0.15.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c47318cd3a61401452de11282242937e3e057c4fd3dbaf601e269d0928a06c0a", size = 729847, upload-time = "2026-08-07T10:48:59.231Z" }, + { url = "https://files.pythonhosted.org/packages/86/ac/fa4d7a424665040e95baf480a6d523446057684b6758624c85338e8a23b2/librt-0.15.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a56a1d4f859a82ca5b99fc4b82c9b027b15e3c455c5cd99e7d0719f27bb20b6c", size = 742736, upload-time = "2026-08-07T10:49:01.151Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/e17a9bb5de6fb8c3186ed1a7d68d21618b027ac2d3633e03d3b6109c67ae/librt-0.15.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:077471b3182db4e17c36ae91555f36a4d2c00080b267f749bcad34a478a9a302", size = 763454, upload-time = "2026-08-07T10:49:03.039Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ec/ecd02cd30935b931b9cdbfed6ab5a099c51b280b4e7baa274da80978ed27/librt-0.15.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:411ca4d1b905b860ceba7570dd6717a71dedaddcc4b0f77ece710aa41ee11f8d", size = 743296, upload-time = "2026-08-07T10:49:04.941Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b5/b3c2b8353ce820a4854f78d19321344242f89fa71c975b71132ba9bf242a/librt-0.15.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1256589e0b0adb31751d685a68bce29d73407ddf4ef05d4188f49d5dcf9566d9", size = 756217, upload-time = "2026-08-07T10:49:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/3c/52/6cc22542ba59146b05cca2a656f9ff8bb67e38e63d12c3b0cc183d837bf1/librt-0.15.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:f42b74a53e5f26a0ba0007411a7455b66c67ce4022a39cc1f56fc4efd65bcbab", size = 741934, upload-time = "2026-08-07T10:49:08.839Z" }, + { url = "https://files.pythonhosted.org/packages/40/32/a04b72b1aa86e3be23b2ecff8c1aad2dcc955bd3956d6d26e7e34267e57a/librt-0.15.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:291bf73caf78b9e88d6fae9bfd693207ff7d832e2fdbe2cf8e746bc13f5f892b", size = 783763, upload-time = "2026-08-07T10:49:10.661Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f0/89eb11dffbe9279ff37144dec786927314502ae0b114f1449dc78c458aab/librt-0.15.0-cp315-cp315t-win32.whl", hash = "sha256:c16d15ee371643ab48dc8248a3e680ebbeca573a13af2c3dd0c985b142d77162", size = 104313, upload-time = "2026-08-07T10:49:12.305Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4a/1f1978c200f563beda63c36adff2d65bbecb81e365e8e69e572f5f70fbc6/librt-0.15.0-cp315-cp315t-win_amd64.whl", hash = "sha256:dbd605739f228912dc49027cb764456b9757750bdc2b6b7773164db7096c6fd1", size = 126889, upload-time = "2026-08-07T10:49:13.881Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/800800bfed7b1fb10fc3f3d557785c3854e80d3f7a9800d784b176a1fc2d/librt-0.15.0-cp315-cp315t-win_arm64.whl", hash = "sha256:84d244b00604d17df3fc7736c327892d6bba66181254aa4087be807b6c342bdc", size = 110700, upload-time = "2026-08-07T10:49:15.499Z" }, +] + [[package]] name = "linecheck" version = "0.1.0" @@ -2013,6 +2132,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] +[[package]] +name = "mypy" +version = "1.20.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/af/e3d4b3e9ec91a0ff9aabfdb38692952acf49bbb899c2e4c29acb3a6da3ae/mypy-1.20.2.tar.gz", hash = "sha256:e8222c26daaafd9e8626dec58ae36029f82585890589576f769a650dd20fd665", size = 3817349, upload-time = "2026-04-21T17:12:28.473Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/4d/9ebeae211caccbdaddde7ed5e31dfcf57faac66be9b11deb1dc6526c8078/mypy-1.20.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4077797a273e56e8843d001e9dfe4ba10e33323d6ade647ff260e5cd97d9758c", size = 14371307, upload-time = "2026-04-21T17:08:56.442Z" }, + { url = "https://files.pythonhosted.org/packages/95/d7/93473d34b61f04fac1aecc01368485c89c5c4af7a4b9a0cab5d77d04b63f/mypy-1.20.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cdecf62abcc4292500d7858aeae87a1f8f1150f4c4dd08fb0b336ee79b2a6df3", size = 13258917, upload-time = "2026-04-21T17:05:50.978Z" }, + { url = "https://files.pythonhosted.org/packages/e2/30/3dd903e8bafb7b5f7bf87fcd58f8382086dea2aa19f0a7b357f21f63071b/mypy-1.20.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c566c3a88b6ece59b3d70f65bedef17304f48eb52ff040a6a18214e1917b3254", size = 13700516, upload-time = "2026-04-21T17:11:33.161Z" }, + { url = "https://files.pythonhosted.org/packages/07/05/c61a140aba4c729ac7bc99ae26fc627c78a6e08f5b9dd319244ea71a3d7e/mypy-1.20.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0deb80d062b2479f2c87ae568f89845afc71d11bc41b04179e58165fd9f31e98", size = 14562889, upload-time = "2026-04-21T17:05:27.674Z" }, + { url = "https://files.pythonhosted.org/packages/fd/87/da78243742ffa8a36d98c3010f0d829f93d5da4e6786f1a1a6f2ad616502/mypy-1.20.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bba9ad231e92a3e424b3e56b65aa17704993425bba97e302c832f9466bb85bac", size = 14803844, upload-time = "2026-04-21T17:10:06.2Z" }, + { url = "https://files.pythonhosted.org/packages/37/52/10a1ddf91b40f843943a3c6db51e2df59c9e237f29d355e95eaab427461f/mypy-1.20.2-cp311-cp311-win_amd64.whl", hash = "sha256:baf593f2765fa3a6b1ef95807dbaa3d25b594f6a52adcc506a6b9cb115e1be67", size = 10846300, upload-time = "2026-04-21T17:12:23.886Z" }, + { url = "https://files.pythonhosted.org/packages/20/02/f9a4415b664c53bd34d6709be59da303abcae986dc4ac847b402edb6fa1e/mypy-1.20.2-cp311-cp311-win_arm64.whl", hash = "sha256:20175a1c0f49863946ec20b7f63255768058ac4f07d2b9ded6a6b46cfb5a9100", size = 9779498, upload-time = "2026-04-21T17:09:23.695Z" }, + { url = "https://files.pythonhosted.org/packages/71/4e/7560e4528db9e9b147e4c0f22660466bf30a0a1fe3d63d1b9d3b0fd354ee/mypy-1.20.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4dbfcf869f6b0517f70cf0030ba6ea1d6645e132337a7d5204a18d8d5636c02b", size = 14539393, upload-time = "2026-04-21T17:07:12.52Z" }, + { url = "https://files.pythonhosted.org/packages/32/d9/34a5efed8124f5a9234f55ac6a4ced4201e2c5b81e1109c49ad23190ec8c/mypy-1.20.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b6481b228d072315b053210b01ac320e1be243dc17f9e5887ef167f23f5fae4", size = 13361642, upload-time = "2026-04-21T17:06:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/d1/14/eb377acf78c03c92d566a1510cda8137348215b5335085ef662ab82ecd3a/mypy-1.20.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34397cdced6b90b836e38182076049fdb41424322e0b0728c946b0939ebdf9f6", size = 13740347, upload-time = "2026-04-21T17:12:04.73Z" }, + { url = "https://files.pythonhosted.org/packages/b9/94/7e4634a32b641aa1c112422eed1bbece61ee16205f674190e8b536f884de/mypy-1.20.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5da6976f20cae27059ea8d0c86e7cef3de720e04c4bb9ee18e3690fdb792066", size = 14734042, upload-time = "2026-04-21T17:07:43.16Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f3/f7e62395cb7f434541b4491a01149a4439e28ace4c0c632bbf5431e92d1f/mypy-1.20.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:56908d7e08318d39f85b1f0c6cfd47b0cac1a130da677630dac0de3e0623e102", size = 14964958, upload-time = "2026-04-21T17:11:00.665Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0d/47e3c3a0ec2a876e35aeac365df3cac7776c36bbd4ed18cc521e1b9d255b/mypy-1.20.2-cp312-cp312-win_amd64.whl", hash = "sha256:d52ad8d78522da1d308789df651ee5379088e77c76cb1994858d40a426b343b9", size = 10911340, upload-time = "2026-04-21T17:10:49.179Z" }, + { url = "https://files.pythonhosted.org/packages/d6/b2/6c852d72e0ea8b01f49da817fb52539993cde327e7d010e0103dc12d0dac/mypy-1.20.2-cp312-cp312-win_arm64.whl", hash = "sha256:785b08db19c9f214dc37d65f7c165d19a30fcecb48abfa30f31b01b5acaabb58", size = 9833947, upload-time = "2026-04-21T17:09:05.267Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c4/b93812d3a192c9bcf5df405bd2f30277cd0e48106a14d1023c7f6ed6e39b/mypy-1.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:edfbfca868cdd6bd8d974a60f8a3682f5565d3f5c99b327640cedd24c4264026", size = 14524670, upload-time = "2026-04-21T17:10:30.737Z" }, + { url = "https://files.pythonhosted.org/packages/f3/47/42c122501bff18eaf1e8f457f5c017933452d8acdc52918a9f59f6812955/mypy-1.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e2877a02380adfcdbc69071a0f74d6e9dbbf593c0dc9d174e1f223ffd5281943", size = 13336218, upload-time = "2026-04-21T17:08:44.069Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/75bbc92f41725fbd585fb17b440b1119b576105df1013622983e18640a93/mypy-1.20.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7488448de6007cd5177c6cea0517ac33b4c0f5ee9b5e9f2be51ce75511a85517", size = 13724906, upload-time = "2026-04-21T17:08:01.02Z" }, + { url = "https://files.pythonhosted.org/packages/a1/32/4c49da27a606167391ff0c39aa955707a00edc500572e562f7c36c08a71f/mypy-1.20.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb9c2fa06887e21d6a3a868762acb82aec34e2c6fd0174064f27c93ede68ad15", size = 14726046, upload-time = "2026-04-21T17:11:22.354Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fc/4e354a1bd70216359deb0c9c54847ee6b32ef78dfb09f5131ff99b494078/mypy-1.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d56a78b646f2e3daa865bc70cd5ec5a46c50045801ca8ff17a0c43abc97e3ee", size = 14955587, upload-time = "2026-04-21T17:12:16.033Z" }, + { url = "https://files.pythonhosted.org/packages/62/b2/c0f2056e9eb8f08c62cafd9715e4584b89132bdc832fcf85d27d07b5f3e5/mypy-1.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:2a4102b03bb7481d9a91a6da8d174740c9c8c4401024684b9ca3b7cc5e49852f", size = 10922681, upload-time = "2026-04-21T17:06:35.842Z" }, + { url = "https://files.pythonhosted.org/packages/e5/14/065e333721f05de8ef683d0aa804c23026bcc287446b61cac657b902ccac/mypy-1.20.2-cp313-cp313-win_arm64.whl", hash = "sha256:a95a9248b0c6fd933a442c03c3b113c3b61320086b88e2c444676d3fd1ca3330", size = 9830560, upload-time = "2026-04-21T17:07:51.023Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d1/b4ec96b0ecc620a4443570c6e95c867903428cfcde4206518eafdd5880c3/mypy-1.20.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:419413398fe250aae057fd2fe50166b61077083c9b82754c341cf4fd73038f30", size = 14524561, upload-time = "2026-04-21T17:06:27.325Z" }, + { url = "https://files.pythonhosted.org/packages/3a/63/d2c2ff4fa66bc49477d32dfa26e8a167ba803ea6a69c5efb416036909d30/mypy-1.20.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e73c07f23009962885c197ccb9b41356a30cc0e5a1d0c2ea8fd8fb1362d7f924", size = 13363883, upload-time = "2026-04-21T17:11:11.239Z" }, + { url = "https://files.pythonhosted.org/packages/2a/56/983916806bf4eddeaaa2c9230903c3669c6718552a921154e1c5182c701f/mypy-1.20.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c64e5973df366b747646fc98da921f9d6eba9716d57d1db94a83c026a08e0fb", size = 13742945, upload-time = "2026-04-21T17:08:34.181Z" }, + { url = "https://files.pythonhosted.org/packages/19/65/0cd9285ab010ee8214c83d67c6b49417c40d86ce46f1aa109457b5a9b8d7/mypy-1.20.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a65aa591af023864fd08a97da9974e919452cfe19cb146c8a5dc692626445dc", size = 14706163, upload-time = "2026-04-21T17:05:15.51Z" }, + { url = "https://files.pythonhosted.org/packages/94/97/48ff3b297cafcc94d185243a9190836fb1b01c1b0918fff64e941e973cc9/mypy-1.20.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fef51b01e638974a6e69885687e9bd40c8d1e09a6cd291cca0619625cf1f558", size = 14938677, upload-time = "2026-04-21T17:05:39.562Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a1/1b4233d255bdd0b38a1f284feeb1c143ca508c19184964e22f8d837ec851/mypy-1.20.2-cp314-cp314-win_amd64.whl", hash = "sha256:913485a03f1bcf5d279409a9d2b9ed565c151f61c09f29991e5faa14033da4c8", size = 11089322, upload-time = "2026-04-21T17:06:44.29Z" }, + { url = "https://files.pythonhosted.org/packages/78/c2/ce7ee2ba36aeb954ba50f18fa25d9c1188578654b97d02a66a15b6f09531/mypy-1.20.2-cp314-cp314-win_arm64.whl", hash = "sha256:c3bae4f855d965b5453784300c12ffc63a548304ac7f99e55d4dc7c898673aa3", size = 10017775, upload-time = "2026-04-21T17:07:20.732Z" }, + { url = "https://files.pythonhosted.org/packages/4e/a1/9d93a7d0b5859af0ead82b4888b46df6c8797e1bc5e1e262a08518c6d48e/mypy-1.20.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2de3dcea53babc1c3237a19002bc3d228ce1833278f093b8d619e06e7cc79609", size = 15549002, upload-time = "2026-04-21T17:08:23.107Z" }, + { url = "https://files.pythonhosted.org/packages/00/d2/09a6a10ee1bf0008f6c144d9676f2ca6a12512151b4e0ad0ff6c4fac5337/mypy-1.20.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:52b176444e2e5054dfcbcb8c75b0b719865c96247b37407184bbfca5c353f2c2", size = 14401942, upload-time = "2026-04-21T17:07:31.837Z" }, + { url = "https://files.pythonhosted.org/packages/57/da/9594b75c3c019e805250bed3583bdf4443ff9e6ef08f97e39ae308cb06f2/mypy-1.20.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:688c3312e5dadb573a2c69c82af3a298d43ecf9e6d264e0f95df960b5f6ac19c", size = 15041649, upload-time = "2026-04-21T17:09:34.653Z" }, + { url = "https://files.pythonhosted.org/packages/97/77/f75a65c278e6e8eba2071f7f5a90481891053ecc39878cc444634d892abe/mypy-1.20.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29752dbbf8cc53f89f6ac096d363314333045c257c9c75cbd189ca2de0455744", size = 15864588, upload-time = "2026-04-21T17:11:44.936Z" }, + { url = "https://files.pythonhosted.org/packages/d7/46/1a4e1c66e96c1a3246ddf5403d122ac9b0a8d2b7e65730b9d6533ba7a6d3/mypy-1.20.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:803203d2b6ea644982c644895c2f78b28d0e208bba7b27d9b921e0ec5eb207c6", size = 16093956, upload-time = "2026-04-21T17:10:17.683Z" }, + { url = "https://files.pythonhosted.org/packages/5a/2c/78a8851264dec38cd736ca5b8bc9380674df0dd0be7792f538916157716c/mypy-1.20.2-cp314-cp314t-win_amd64.whl", hash = "sha256:9bcb8aa397ff0093c824182fd76a935a9ba7ad097fcbef80ae89bf6c1731d8ec", size = 12568661, upload-time = "2026-04-21T17:11:54.473Z" }, + { url = "https://files.pythonhosted.org/packages/83/01/cd7318aa03493322ce275a0e14f4f52b8896335e4e79d4fb8153a7ad2b77/mypy-1.20.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e061b58443f1736f8a37c48978d7ab581636d6ab03e3d4f99e3fa90463bb9382", size = 10389240, upload-time = "2026-04-21T17:09:42.719Z" }, + { url = "https://files.pythonhosted.org/packages/28/9a/f23c163e25b11074188251b0b5a0342625fc1cdb6af604757174fa9acc9b/mypy-1.20.2-py3-none-any.whl", hash = "sha256:a94c5a76ab46c5e6257c7972b6c8cff0574201ca7dc05647e33e795d78680563", size = 2637314, upload-time = "2026-04-21T17:05:54.5Z" }, +] + [[package]] name = "mypy-extensions" version = "1.1.0" @@ -2544,7 +2713,7 @@ models = [ [[package]] name = "policyengine-api" -version = "3.50.0" +version = "3.52.0" source = { editable = "." } dependencies = [ { name = "a2wsgi" }, @@ -2586,6 +2755,7 @@ dependencies = [ dev = [ { name = "build" }, { name = "coverage" }, + { name = "mypy" }, { name = "pytest" }, { name = "pytest-snapshot" }, { name = "pytest-timeout" }, @@ -2613,6 +2783,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.27.0" }, { name = "markupsafe", specifier = ">=3,<4" }, { name = "microdf-python", specifier = ">=1.0.0" }, + { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.15,<2" }, { name = "openai" }, { name = "packaging", specifier = ">=24,<27" }, { name = "policyengine", extras = ["models"], specifier = "==5.2.0" },