feat: run the Hasura database on pgvector and add semantic search

charts/mint/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 9.0.0-beta.12
+version: 9.0.0-beta.13
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.

charts/mint/README.md

@@ -1,6 +1,6 @@
 # MINT
 
-![Version: 9.0.0-beta.12](https://img.shields.io/badge/Version-9.0.0--beta.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.3](https://img.shields.io/badge/AppVersion-1.16.3-informational?style=flat-square)
+![Version: 9.0.0-beta.13](https://img.shields.io/badge/Version-9.0.0--beta.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.3](https://img.shields.io/badge/AppVersion-1.16.3-informational?style=flat-square)
 
 A Helm chart for MINT
 
@@ -146,8 +146,8 @@ A Helm chart for MINT
 | components.hasura_db.arm_image.repository | string | `"imresamu/postgis-arm64"` | Docker image repository for ARM-based Hasura database |
 | components.hasura_db.arm_image.tag | string | `"12-3.4-alpine"` | Docker image tag for ARM-based Hasura database |
 | components.hasura_db.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for Hasura database |
-| components.hasura_db.image.repository | string | `"postgis/postgis"` | Docker image repository for Hasura database |
-| components.hasura_db.image.tag | string | `"10-3.2-alpine"` | Docker image tag for Hasura database |
+| components.hasura_db.image.repository | string | `"ghcr.io/mintproject/postgres-pgvector"` | Docker image repository for Hasura database |
+| components.hasura_db.image.tag | string | `"develop"` | Docker image tag for Hasura database |
 | components.hasura_db.persistence.accessModes[0] | string | `"ReadWriteOnce"` |  |
 | components.hasura_db.persistence.annotations."helm.sh/resource-policy" | string | `"keep"` |  |
 | components.hasura_db.persistence.dataSource | object | `{}` |  |
@@ -244,6 +244,13 @@ A Helm chart for MINT
 | components.model_catalog_explorer.ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` |  |
 | components.model_catalog_explorer.ingress.tls | list | `[]` |  |
 | components.model_catalog_explorer.resources | object | `{}` | Resource specifications for Model Catalog Explorer |
+| components.semantic_search.enabled | bool | `true` | Enable or disable the semantic search service |
+| components.semantic_search.environment.embedding_refresh_seconds | int | `60` | Seconds between embedding refresh passes |
+| components.semantic_search.environment.webhook_url | string | `""` | Webhook URL that the Hasura event triggers call. Defaults to internal service DNS (http://<prefix>-semantic-search:8091/events/catalog) |
+| components.semantic_search.image.pullPolicy | string | `"Always"` | Image pull policy for the semantic search service |
+| components.semantic_search.image.repository | string | `"ghcr.io/mintproject/semantic-search"` | Docker image repository for the semantic search service |
+| components.semantic_search.image.tag | string | `""` | Docker image tag for the semantic search service. Empty by default, so `global.imageTag` applies. |
+| components.semantic_search.resources | object | `{}` | Resource specifications for the semantic search service |
 | components.ui.config.airflow_api | string | `""` |  |
 | components.ui.config.airflow_dag_download_thread_id | string | `""` |  |
 | components.ui.config.execution_component_from_tapis | bool | `false` |  |

charts/mint/templates/hasura.yaml

@@ -245,6 +245,12 @@ spec:
                 key: admin_secret
           - name: DB_HOST
             value: {{ include "mint.prefix" . }}-hasura-db
+          # Nine event triggers in the metadata declare
+          # `webhook_from_env: SVO_SEMANTIC_SEARCH_WEBHOOK_URL`. Hasura rejects
+          # the whole `metadata apply` when the variable is absent, so set it
+          # even when `components.semantic_search.enabled` is false.
+          - name: SVO_SEMANTIC_SEARCH_WEBHOOK_URL
+            value: {{ .Values.components.semantic_search.environment.webhook_url | default (printf "http://%s-semantic-search:8091/events/catalog" (include "mint.prefix" .)) | quote }}
           ports:
             - name: http
               containerPort: 8080

charts/mint/templates/semantic-search.yaml

@@ -0,0 +1,89 @@
+---
+{{ if .Values.components.semantic_search.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "mint.prefix" . }}-semantic-search
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "helm.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: "http"
+      port: 8091
+      targetPort: 8091
+      protocol: TCP
+  selector:
+    app: {{ include "mint.prefix" . }}-semantic-search
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "mint.prefix" . }}-semantic-search
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ include "mint.prefix" . }}-semantic-search
+  template:
+    metadata:
+      labels:
+        app: {{ include "mint.prefix" . }}-semantic-search
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      # The service reads and writes the Hasura database directly. Wait for it,
+      # the same way the Hasura deployment does.
+      initContainers:
+      - name: wait-for-db
+        image: postgres:13-alpine
+        command: ['sh', '-c',
+          'until pg_isready -h {{ include "mint.prefix" . }}-hasura-db -U $POSTGRES_USER; do echo waiting for database; sleep 2; done;']
+        env:
+          - name: POSTGRES_USER
+            valueFrom:
+              secretKeyRef:
+                name: {{ include "mint.prefix" . }}-hasura-secrets
+                key: username
+      containers:
+        - name: semantic-search
+          {{- with .Values.components.semantic_search }}
+          image: "{{ .image.repository }}:{{ .image.tag | default $.Values.global.imageTag | default $.Chart.AppVersion }}"
+          imagePullPolicy: {{ .image.pullPolicy }}
+          resources:
+            {{- toYaml .resources | nindent 12 }}
+          ports:
+            - name: http
+              containerPort: 8091
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+          env:
+            - name: SVO_EMBEDDING_REFRESH_SECONDS
+              value: "{{ .environment.embedding_refresh_seconds }}"
+          {{- end }}
+            # libpq accepts the `postgres://` scheme that this secret carries.
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "mint.prefix" . }}-hasura-secrets
+                  key: database_url
+{{ end }}

charts/mint/values.yaml

@@ -308,13 +308,29 @@ components:
               pathType: ImplementationSpecific
       tls: []
   hasura_db:
+    # The image carries PostGIS *and* pgvector. Migration
+    # `1771200021000_standard_variable_embeddings` runs `CREATE EXTENSION vector`.
+    # `postgis/postgis` ships no pgvector, so every `hasura migrate apply` failed
+    # with `could not open extension control file ".../vector.control"`.
+    #
+    # BREAKING: this image is PostgreSQL 16. The previous default was PostgreSQL
+    # 10. PostgreSQL 16 refuses to start on PostgreSQL 10 data files. An existing
+    # deployment must do this before the upgrade:
+    #   1. Scale the Hasura deployment to 0.
+    #   2. Dump the database with a PostgreSQL 16 `pg_dump`.
+    #   3. Delete the PVC `data-<release>-hasura-db-0`.
+    #   4. Upgrade, then restore the dump with `pg_restore`.
+    # The PVC carries `helm.sh/resource-policy: keep`, so Helm never deletes it.
     image:
       # -- Docker image repository for Hasura database
-      repository: postgis/postgis
+      repository: ghcr.io/mintproject/postgres-pgvector
       # -- Docker image tag for Hasura database
-      tag: 10-3.2-alpine
+      tag: develop
       # -- Image pull policy for Hasura database
       pullPolicy: IfNotPresent
+    # ARM has no pgvector image yet. `mintproject/monorepo` builds
+    # `postgres-pgvector` for linux/amd64 only. An ARM deployment keeps the old
+    # PostGIS image and cannot apply the embedding migration.
     arm_image:
       # -- Docker image repository for ARM-based Hasura database
       repository: imresamu/postgis-arm64
@@ -338,6 +354,32 @@ components:
       selector: {}
       annotations:
         helm.sh/resource-policy: keep
+  # The semantic search service. Hasura event triggers post catalog changes to
+  # it, and it writes the embeddings back into `modelcatalog_standard_variable`.
+  #
+  # `SVO_SEMANTIC_SEARCH_WEBHOOK_URL` is always set on the Hasura pod, even when
+  # this component is disabled. The metadata in `mintproject/monorepo` declares
+  # nine event triggers with `webhook_from_env: SVO_SEMANTIC_SEARCH_WEBHOOK_URL`.
+  # Hasura rejects the whole `metadata apply` when the variable is absent.
+  semantic_search:
+    # -- Enable or disable the semantic search service
+    enabled: true
+    image:
+      # -- Docker image repository for the semantic search service
+      repository: ghcr.io/mintproject/semantic-search
+      # -- Docker image tag for the semantic search service. Empty by default, so
+      # `global.imageTag` applies.
+      tag: ""
+      # -- Image pull policy for the semantic search service
+      pullPolicy: Always
+    # -- Resource specifications for the semantic search service
+    resources: {}
+    environment:
+      # -- Seconds between embedding refresh passes
+      embedding_refresh_seconds: 60
+      # -- Webhook URL that the Hasura event triggers call. Defaults to internal
+      # service DNS (http://<prefix>-semantic-search:8091/events/catalog)
+      webhook_url: ""
   model_catalog_explorer:
     # -- Enable or disable Model Catalog Explorer
     enabled: true