Fix and improve probes (#79)

* Fix probes when Qdrant is served over TLS This also improves the integration test runtime and reliability by not installing a fresh cluster for every test but upgrading it with the new config. Fixes https://github.com/qdrant/qdrant-helm/issues/77 * Deactivate liveness and startup probes by default Since these probes restart the pod on failure, they can disrupt the health of the Qdrant cluster unnecessarily and cause more problems then actually help. Having a readinessProbe is usually enough. * Fix tests * Try to fix connection issues
2026-01-27 10:20:18 +00:00 · 2023-09-29 12:31:54 +02:00
parent 52515f661e
commit 264b4cfe96
13 changed files with 126 additions and 95 deletions
--- a/charts/qdrant/templates/statefulset.yaml
+++ b/charts/qdrant/templates/statefulset.yaml
@@ -100,6 +100,9 @@ spec:
            httpGet:
              path: /livez
              port: {{ .targetPort }}
+              {{- if and $values.config.service $values.config.service.enable_tls }}
+              scheme: HTTPS
+              {{- end }}
            {{- end }}
            initialDelaySeconds: {{ $values.livenessProbe.initialDelaySeconds }}
            timeoutSeconds: {{ $values.livenessProbe.timeoutSeconds }}
@@ -117,6 +120,9 @@ spec:
            httpGet:
              path: /readyz
              port: {{ .targetPort }}
+              {{- if and $values.config.service $values.config.service.enable_tls }}
+              scheme: HTTPS
+              {{- end }}
            {{- end }}
            initialDelaySeconds: {{ $values.readinessProbe.initialDelaySeconds }}
            timeoutSeconds: {{ $values.readinessProbe.timeoutSeconds }}
@@ -134,6 +140,9 @@ spec:
            httpGet:
              path: /readyz
              port: {{ .targetPort }}
+              {{- if and $values.config.service $values.config.service.enable_tls }}
+              scheme: HTTPS
+              {{- end }}
            {{- end }}
            initialDelaySeconds: {{ $values.startupProbe.initialDelaySeconds }}
            timeoutSeconds: {{ $values.startupProbe.timeoutSeconds }}
--- a/charts/qdrant/templates/tests/test-db-interaction.yaml
+++ b/charts/qdrant/templates/tests/test-db-interaction.yaml
@@ -11,15 +11,21 @@ metadata:
 spec:
  containers:
    - name: test-script
-      image: registry.suse.com/bci/python:3.11
+      image: registry.suse.com/bci/bci-base:latest
      args: ['bash', '/app/entrypoint.sh']
      volumeMounts:
        - mountPath: /app
          name: test-script
+        {{- if .Values.additionalVolumeMounts }}
+{{- toYaml .Values.additionalVolumeMounts | default "" | nindent 8 }}
+        {{- end}}
  volumes:
    - name: test-script
      configMap:
        name: "{{ include "qdrant.fullname" . }}-test-db-interaction"
+    {{- if .Values.additionalVolumes }}
+{{- toYaml .Values.additionalVolumes | default "" | nindent 4 }}
+    {{- end}}
  restartPolicy: Never
  serviceAccountName: {{ include "qdrant.fullname" . }}
 ---
@@ -35,43 +41,57 @@ data:
  entrypoint.sh: |
    #!/bin/bash
    set -xe
-    zypper install -y python311-dbm
-    pip3 install qdrant-client=={{ .Chart.AppVersion }}
-    python3 /app/test.py
-  test.py: |
-    from qdrant_client import QdrantClient
-    from qdrant_client.http.models import Distance, VectorParams
-    from qdrant_client.http.models import PointStruct
+    echo 'connect-timeout = 5' > $HOME/.curlrc
+    echo 'retry = 5' >> $HOME/.curlrc

+    if [ -d /mnt/secrets/certs ]; then
+      cp /mnt/secrets/certs/ca.pem /usr/share/pki/trust/anchors/private-ca.pem
+      update-ca-certificates
+    fi
+
+    QDRANT_COLLECTION="test_collection"
    {{- range .Values.service.ports }}
    {{- if eq .name "http" }}
-    print("Connecting to {{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}")
-    client = QdrantClient("{{ include "qdrant.fullname" $root }}.{{ $namespace }}", port={{ .port }})
+    echo "Connecting to {{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}"
+    QDRANT_URL="http://{{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}"
+    {{- if and $root.Values.config.service $root.Values.config.service.enable_tls }}
+    echo "Using https"
+    QDRANT_URL="https://{{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}"
    {{- end }}
    {{- end }}
+    {{- end }}
+    API_KEY_HEADER=""
+    {{- if .Values.apiKey }}
+    API_KEY_HEADER="Api-key: {{ .Values.apiKey }}"
+    {{- end }}

-    client.recreate_collection(
-        collection_name="test_collection",
-        vectors_config=VectorParams(size=4, distance=Distance.DOT),
-    )
+    # Delete collection if exists
+    curl -X DELETE -H "${API_KEY_HEADER}" $QDRANT_URL/collections/${QDRANT_COLLECTION}

-    operation_info = client.upsert(
-      collection_name="test_collection",
-      wait=True,
-      points=[
-        PointStruct(id=1, vector=[0.05, 0.61, 0.76, 0.74], payload={"city": "Berlin"}),
-        PointStruct(id=2, vector=[0.19, 0.81, 0.75, 0.11], payload={"city": "London"}),
-        PointStruct(id=3, vector=[0.36, 0.55, 0.47, 0.94], payload={"city": "Moscow"}),
-        PointStruct(id=4, vector=[0.18, 0.01, 0.85, 0.80], payload={"city": "New York"}),
-        PointStruct(id=5, vector=[0.24, 0.18, 0.22, 0.44], payload={"city": "Beijing"}),
-        PointStruct(id=6, vector=[0.35, 0.08, 0.11, 0.44], payload={"city": "Mumbai"}),
-        ]
-    )
-    print(operation_info)
+    # Create collection
+    curl -X PUT \
+    -H 'Content-Type: application-json' \
+    -d '{"vectors":{"size":4,"distance":"Dot"}}' \
+    -H  "${API_KEY_HEADER}" \
+    $QDRANT_URL/collections/${QDRANT_COLLECTION} --fail-with-body

-    search_result = client.search(
-      collection_name="test_collection",
-      query_vector=[0.2, 0.1, 0.9, 0.7],
-      limit=3
-    )
-    print(search_result)
+    # Insert points
+    curl -X PUT \
+    -H 'Content-Type: application-json' \
+    -d '{"points":[
+      {"id":1,"vector":[0.05, 0.61, 0.76, 0.74],"payload":{"city":"Berlin"}},
+      {"id":2,"vector":[0.19, 0.81, 0.75, 0.11],"payload":{"city":"London"}},
+      {"id":3,"vector":[0.36, 0.55, 0.47, 0.94],"payload":{"city":"Moscow"}},
+      {"id":4,"vector":[0.18, 0.01, 0.85, 0.80],"payload":{"city":"New York"}},
+      {"id":5,"vector":[0.24, 0.18, 0.22, 0.44],"payload":{"city":"Beijing"}},
+      {"id":6,"vector":[0.35, 0.08, 0.11, 0.44],"payload":{"city":"Mumbai"}}
+    ]}' \
+    -H  "${API_KEY_HEADER}" \
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points --fail-with-body
+
+    # Run query
+    curl -X POST \
+    -H 'Content-Type: application-json' \
+    -d '{"vector":[0.2, 0.1, 0.9, 0.7],"limit":3}' \
+    -H  "${API_KEY_HEADER}" \
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points/search --fail-with-body
--- a/charts/qdrant/values.yaml
+++ b/charts/qdrant/values.yaml
@@ -54,7 +54,7 @@ ingress:
    #   secretName: tls-secret-name

 livenessProbe:
-  enabled: true
+  enabled: false
  initialDelaySeconds: 5
  periodSeconds: 5
  timeoutSeconds: 1
@@ -70,7 +70,7 @@ readinessProbe:
  successThreshold: 1

 startupProbe:
-  enabled: true
+  enabled: false
  initialDelaySeconds: 10
  periodSeconds: 5
  timeoutSeconds: 1
--- a/test/integration/api_key.bats
+++ b/test/integration/api_key.bats
@@ -1,20 +1,12 @@
 setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-    helm install qdrant charts/qdrant --set apiKey=foobar -n qdrant-helm-integration --wait
+    helm upgrade --install qdrant charts/qdrant --set apiKey=foobar -n qdrant-helm-integration --wait
    kubectl rollout status statefulset qdrant -n qdrant-helm-integration
 }

-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
-}
-
@test "api key authentication works" {
-    run kubectl exec -n default curl -- curl -s http://qdrant.qdrant-helm-integration:6333/collections -H 'api-key: foobar'
+    run kubectl exec -n default curl -- curl -s http://qdrant.qdrant-helm-integration:6333/collections -H 'api-key: foobar' --fail-with-body
    [ $status -eq 0 ]
-    [ "${output}" != "Invalid api-key" ]
+    [[ "${output}" =~ .*\"status\":\"ok\".* ]]
 }

@test "api key authentication fails with key" {
--- a/test/integration/assets/.gitignore
+++ b/test/integration/assets/.gitignore
@@ -0,0 +1,6 @@
+*.key
+*.crt
+*.csr
+*.pem
+*.ext
+*.srl
--- a/test/integration/assets/tls-values.yaml
+++ b/test/integration/assets/tls-values.yaml
@@ -0,0 +1,19 @@
+additionalVolumes:
+  - secret:
+      secretName: test-tls
+    name: tls
+
+additionalVolumeMounts:
+  - name: tls
+    mountPath: /mnt/secrets/certs
+
+config:
+  service:
+    enable_tls: true
+  tls:
+    cert: "/mnt/secrets/certs/tls.crt"
+    key: "/mnt/secrets/certs/tls.key"
+    ca_cert: "/mnt/secrets/certs/ca.pem"
+  cluster:
+    p2p:
+      enable_tls: true
--- a/test/integration/default_installation.bats
+++ b/test/integration/default_installation.bats
@@ -1,17 +1,6 @@
 setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-    helm install qdrant charts/qdrant -n qdrant-helm-integration --wait
+    helm upgrade --install qdrant charts/qdrant -n qdrant-helm-integration --wait
    kubectl rollout status statefulset qdrant -n qdrant-helm-integration
-    # wait a bit to ensure that qdrant is really up, and we don't get connection refused
-    # errors in Github Actions
-    sleep 10
-}
-
-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
 }

@test "helm test - default values" {
--- a/test/integration/https.bats
+++ b/test/integration/https.bats
@@ -0,0 +1,24 @@
+setup_file() {
+    rm test/integration/assets/ca.* || true
+    rm test/integration/assets/tls.* || true
+    openssl genrsa -des3 -passout pass:insecure -out test/integration/assets/ca.key 2048
+    openssl req -x509 -passin pass:insecure -new -nodes -key test/integration/assets/ca.key -sha256 -days 1825 -out test/integration/assets/ca.pem -subj "/C=DE/ST=Berlin/L=Berlin/O=Qdrant/OU=Cloud/CN=qdrant"
+    openssl genrsa -out test/integration/assets/tls.key 2048
+    openssl req -new -key test/integration/assets/tls.key -out test/integration/assets/tls.csr -subj "/C=DE/ST=Berlin/L=Berlin/O=Qdrant/OU=Cloud/CN=qdrant.qdrant-helm-integration"
+    openssl x509 -req -passin pass:insecure -in test/integration/assets/tls.csr -CA test/integration/assets/ca.pem -CAkey test/integration/assets/ca.key -CAcreateserial -out test/integration/assets/tls.crt -days 825 -sha256
+
+    kubectl -n qdrant-helm-integration create secret generic test-tls --from-file=tls.key="test/integration/assets/tls.key" --from-file=tls.crt="test/integration/assets/tls.crt" --from-file=ca.pem="test/integration/assets/ca.pem"
+    helm upgrade --install qdrant charts/qdrant -n qdrant-helm-integration --wait -f test/integration/assets/tls-values.yaml
+    kubectl rollout status statefulset qdrant -n qdrant-helm-integration
+}
+
+@test "Connection with https" {
+    run kubectl exec -n default curl -- curl -k -s https://qdrant.qdrant-helm-integration:6333/collections --fail-with-body
+    [ $status -eq 0 ]
+    [[ "${output}" =~ .*\"status\":\"ok\".* ]]
+}
+
+@test "helm test - with https" {
+    run helm test qdrant -n qdrant-helm-integration --logs
+    [ $status -eq 0 ]
+}
--- a/test/integration/random_api_key.bats
+++ b/test/integration/random_api_key.bats
@@ -1,22 +1,14 @@
 setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-    helm install qdrant charts/qdrant --set apiKey=true -n qdrant-helm-integration --wait
+    helm upgrade --install qdrant charts/qdrant --set apiKey=true -n qdrant-helm-integration --wait
    kubectl rollout status statefulset qdrant -n qdrant-helm-integration
 }

-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
-}
-
@test "random api key works" {
    apiKey=$(kubectl -n qdrant-helm-integration get secret qdrant-apikey -o jsonpath="{.data.api-key}" | base64 --decode)
    [ ${#apiKey} -eq 32 ]
-    run kubectl exec -n default curl -- curl -s http://qdrant.qdrant-helm-integration:6333/collections -H "api-key: ${apiKey}"
+    run kubectl exec -n default curl -- curl -s http://qdrant.qdrant-helm-integration:6333/collections -H "api-key: ${apiKey}" --fail-with-body
    [ $status -eq 0 ]
-    [ "${output}" != "Invalid api-key" ]
+    [[ "${output}" =~ .*\"status\":\"ok\".* ]]
 }

@test "random api key stays the same after upgrade" {
--- a/test/integration/security_context.bats
+++ b/test/integration/security_context.bats
@@ -1,16 +1,5 @@
-setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-}
-
-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
-}
-
@test "update without security context to security context corrects file permissions" {
-    helm install qdrant charts/qdrant --set-json 'podSecurityContext=false,containerSecurityContext=false' -n qdrant-helm-integration --wait
+    helm upgrade --install qdrant charts/qdrant --set-json 'podSecurityContext=false,containerSecurityContext=false' -n qdrant-helm-integration --wait
    kubectl rollout status statefulset qdrant -n qdrant-helm-integration
    user=$(kubectl exec qdrant-0 -n qdrant-helm-integration -- id -u)
    [ "${user}" = "0" ]
--- a/test/integration/setup_suite.bash
+++ b/test/integration/setup_suite.bash
@@ -3,6 +3,8 @@ setup_suite() {
    kubectl create serviceaccount default -n default
    kubectl -n default run curl --image=curlimages/curl --command -- sh -c "sleep 3600"
    kubectl wait --for=condition=Ready pod/curl -n default --timeout=300s
+    kubectl create namespace qdrant-helm-integration
+    kubectl create serviceaccount default -n qdrant-helm-integration || true
 }

 teardown_suite() {
--- a/test/integration/unprivileged_image.bats
+++ b/test/integration/unprivileged_image.bats
@@ -1,17 +1,6 @@
 setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-    helm install qdrant charts/qdrant -n qdrant-helm-integration --wait --set image.useUnprivilegedImage=true
+    helm upgrade --install qdrant charts/qdrant -n qdrant-helm-integration --wait --set image.useUnprivilegedImage=true
    kubectl rollout status statefulset qdrant -n qdrant-helm-integration
-    # wait a bit to ensure that qdrant is really up, and we don't get connection refused
-    # errors in Github Actions
-    sleep 10
-}
-
-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
 }

@test "helm test - with unprivileged image" {
--- a/test/qdrant_probes_test.go
+++ b/test/qdrant_probes_test.go
@@ -38,7 +38,7 @@ func TestDefaultProbesOnStatefulset(t *testing.T) {
 		return container.Name == "qdrant"
 	})

-	require.Equal(t, "/readyz", container.StartupProbe.HTTPGet.Path)
+	require.Empty(t, container.StartupProbe)
 	require.Equal(t, "/readyz", container.ReadinessProbe.HTTPGet.Path)
-	require.Equal(t, "/livez", container.LivenessProbe.HTTPGet.Path)
+	require.Empty(t, container.LivenessProbe)
 }