diff --git a/charts/mint/Chart.yaml b/charts/mint/Chart.yaml index d27a67d..4536ad9 100644 --- a/charts/mint/Chart.yaml +++ b/charts/mint/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 9.0.0-beta.12 +version: 9.0.0-beta.13 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. diff --git a/charts/mint/README.md b/charts/mint/README.md index 7c34a70..043b749 100644 --- a/charts/mint/README.md +++ b/charts/mint/README.md @@ -1,6 +1,6 @@ # MINT -![Version: 9.0.0-beta.12](https://img.shields.io/badge/Version-9.0.0--beta.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.3](https://img.shields.io/badge/AppVersion-1.16.3-informational?style=flat-square) +![Version: 9.0.0-beta.13](https://img.shields.io/badge/Version-9.0.0--beta.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.3](https://img.shields.io/badge/AppVersion-1.16.3-informational?style=flat-square) A Helm chart for MINT @@ -146,8 +146,8 @@ A Helm chart for MINT | components.hasura_db.arm_image.repository | string | `"imresamu/postgis-arm64"` | Docker image repository for ARM-based Hasura database | | components.hasura_db.arm_image.tag | string | `"12-3.4-alpine"` | Docker image tag for ARM-based Hasura database | | components.hasura_db.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for Hasura database | -| components.hasura_db.image.repository | string | `"postgis/postgis"` | Docker image repository for Hasura database | -| components.hasura_db.image.tag | string | `"10-3.2-alpine"` | Docker image tag for Hasura database | +| components.hasura_db.image.repository | string | `"ghcr.io/mintproject/postgres-pgvector"` | Docker image repository for Hasura database | +| components.hasura_db.image.tag | string | `"develop"` | Docker image tag for Hasura database | | components.hasura_db.persistence.accessModes[0] | string | `"ReadWriteOnce"` | | | components.hasura_db.persistence.annotations."helm.sh/resource-policy" | string | `"keep"` | | | components.hasura_db.persistence.dataSource | object | `{}` | | @@ -244,6 +244,13 @@ A Helm chart for MINT | components.model_catalog_explorer.ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | | components.model_catalog_explorer.ingress.tls | list | `[]` | | | components.model_catalog_explorer.resources | object | `{}` | Resource specifications for Model Catalog Explorer | +| components.semantic_search.enabled | bool | `true` | Enable or disable the semantic search service | +| components.semantic_search.environment.embedding_refresh_seconds | int | `60` | Seconds between embedding refresh passes | +| components.semantic_search.environment.webhook_url | string | `""` | Webhook URL that the Hasura event triggers call. Defaults to internal service DNS (http://-semantic-search:8091/events/catalog) | +| components.semantic_search.image.pullPolicy | string | `"Always"` | Image pull policy for the semantic search service | +| components.semantic_search.image.repository | string | `"ghcr.io/mintproject/semantic-search"` | Docker image repository for the semantic search service | +| components.semantic_search.image.tag | string | `""` | Docker image tag for the semantic search service. Empty by default, so `global.imageTag` applies. | +| components.semantic_search.resources | object | `{}` | Resource specifications for the semantic search service | | components.ui.config.airflow_api | string | `""` | | | components.ui.config.airflow_dag_download_thread_id | string | `""` | | | components.ui.config.execution_component_from_tapis | bool | `false` | | diff --git a/charts/mint/templates/hasura.yaml b/charts/mint/templates/hasura.yaml index 535a4fd..866be90 100644 --- a/charts/mint/templates/hasura.yaml +++ b/charts/mint/templates/hasura.yaml @@ -245,6 +245,12 @@ spec: key: admin_secret - name: DB_HOST value: {{ include "mint.prefix" . }}-hasura-db + # Nine event triggers in the metadata declare + # `webhook_from_env: SVO_SEMANTIC_SEARCH_WEBHOOK_URL`. Hasura rejects + # the whole `metadata apply` when the variable is absent, so set it + # even when `components.semantic_search.enabled` is false. + - name: SVO_SEMANTIC_SEARCH_WEBHOOK_URL + value: {{ .Values.components.semantic_search.environment.webhook_url | default (printf "http://%s-semantic-search:8091/events/catalog" (include "mint.prefix" .)) | quote }} ports: - name: http containerPort: 8080 diff --git a/charts/mint/templates/semantic-search.yaml b/charts/mint/templates/semantic-search.yaml new file mode 100644 index 0000000..89fa105 --- /dev/null +++ b/charts/mint/templates/semantic-search.yaml @@ -0,0 +1,89 @@ +--- +{{ if .Values.components.semantic_search.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mint.prefix" . }}-semantic-search + namespace: {{ .Release.Namespace }} + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: "http" + port: 8091 + targetPort: 8091 + protocol: TCP + selector: + app: {{ include "mint.prefix" . }}-semantic-search +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mint.prefix" . }}-semantic-search + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "mint.prefix" . }}-semantic-search + template: + metadata: + labels: + app: {{ include "mint.prefix" . }}-semantic-search + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + # The service reads and writes the Hasura database directly. Wait for it, + # the same way the Hasura deployment does. + initContainers: + - name: wait-for-db + image: postgres:13-alpine + command: ['sh', '-c', + 'until pg_isready -h {{ include "mint.prefix" . }}-hasura-db -U $POSTGRES_USER; do echo waiting for database; sleep 2; done;'] + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ include "mint.prefix" . }}-hasura-secrets + key: username + containers: + - name: semantic-search + {{- with .Values.components.semantic_search }} + image: "{{ .image.repository }}:{{ .image.tag | default $.Values.global.imageTag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ .image.pullPolicy }} + resources: + {{- toYaml .resources | nindent 12 }} + ports: + - name: http + containerPort: 8091 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + env: + - name: SVO_EMBEDDING_REFRESH_SECONDS + value: "{{ .environment.embedding_refresh_seconds }}" + {{- end }} + # libpq accepts the `postgres://` scheme that this secret carries. + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "mint.prefix" . }}-hasura-secrets + key: database_url +{{ end }} diff --git a/charts/mint/values.yaml b/charts/mint/values.yaml index a744335..e41fd39 100644 --- a/charts/mint/values.yaml +++ b/charts/mint/values.yaml @@ -308,13 +308,29 @@ components: pathType: ImplementationSpecific tls: [] hasura_db: + # The image carries PostGIS *and* pgvector. Migration + # `1771200021000_standard_variable_embeddings` runs `CREATE EXTENSION vector`. + # `postgis/postgis` ships no pgvector, so every `hasura migrate apply` failed + # with `could not open extension control file ".../vector.control"`. + # + # BREAKING: this image is PostgreSQL 16. The previous default was PostgreSQL + # 10. PostgreSQL 16 refuses to start on PostgreSQL 10 data files. An existing + # deployment must do this before the upgrade: + # 1. Scale the Hasura deployment to 0. + # 2. Dump the database with a PostgreSQL 16 `pg_dump`. + # 3. Delete the PVC `data--hasura-db-0`. + # 4. Upgrade, then restore the dump with `pg_restore`. + # The PVC carries `helm.sh/resource-policy: keep`, so Helm never deletes it. image: # -- Docker image repository for Hasura database - repository: postgis/postgis + repository: ghcr.io/mintproject/postgres-pgvector # -- Docker image tag for Hasura database - tag: 10-3.2-alpine + tag: develop # -- Image pull policy for Hasura database pullPolicy: IfNotPresent + # ARM has no pgvector image yet. `mintproject/monorepo` builds + # `postgres-pgvector` for linux/amd64 only. An ARM deployment keeps the old + # PostGIS image and cannot apply the embedding migration. arm_image: # -- Docker image repository for ARM-based Hasura database repository: imresamu/postgis-arm64 @@ -338,6 +354,32 @@ components: selector: {} annotations: helm.sh/resource-policy: keep + # The semantic search service. Hasura event triggers post catalog changes to + # it, and it writes the embeddings back into `modelcatalog_standard_variable`. + # + # `SVO_SEMANTIC_SEARCH_WEBHOOK_URL` is always set on the Hasura pod, even when + # this component is disabled. The metadata in `mintproject/monorepo` declares + # nine event triggers with `webhook_from_env: SVO_SEMANTIC_SEARCH_WEBHOOK_URL`. + # Hasura rejects the whole `metadata apply` when the variable is absent. + semantic_search: + # -- Enable or disable the semantic search service + enabled: true + image: + # -- Docker image repository for the semantic search service + repository: ghcr.io/mintproject/semantic-search + # -- Docker image tag for the semantic search service. Empty by default, so + # `global.imageTag` applies. + tag: "" + # -- Image pull policy for the semantic search service + pullPolicy: Always + # -- Resource specifications for the semantic search service + resources: {} + environment: + # -- Seconds between embedding refresh passes + embedding_refresh_seconds: 60 + # -- Webhook URL that the Hasura event triggers call. Defaults to internal + # service DNS (http://-semantic-search:8091/events/catalog) + webhook_url: "" model_catalog_explorer: # -- Enable or disable Model Catalog Explorer enabled: true