Increase curl retries to fix CI flakiness (#180)

* Increase curl retries to fix CI flakiness

* Add retries to curl container

charts/qdrant/README.md

@@ -92,6 +92,19 @@ Some storage providers allow resizing volumes in-place, but most require a pod r
 kubectl rollout restart statefulset qdrant
 ```
 
+### Immutable Pod fields
+
+In addition to immutable fields on StatefulSets, Pods also have some fields which are immutable, which means the above method may not work for some changes, such as setting `snapshotPersistence.enabled: true`. In that case, after following the above method, you'll see an error like this when you `kubectl describe` your StatefulSet:
+
+```
+pod updates may not change fields other than `spec.containers[*].image`,
+`spec.initContainers[*].image`,`spec.activeDeadlineSeconds`,
+`spec.tolerations` (only additions to existing tolerations),
+`spec.terminationGracePeriodSeconds` (allow it to be set to 1 if it was previously negative)
+```
+
+To fix this, you must manually delete all of your Qdrant pods, starting with node-0. This will cause your cluster to go down, but will allow the StatefulSet to recreate your Pods with the correct configuration.
+
 ## Restoring from Snapshots
 
 This helm chart allows you to restore a snapshot into your Qdrant cluster either from an internal or external PersistentVolumeClaim.

charts/qdrant/templates/tests/test-db-interaction.yaml

@@ -41,9 +41,15 @@ data:
   entrypoint.sh: |
     #!/bin/bash
     set -xe
+    # Kind's networking is very flaky
     echo 'connect-timeout = 5' > $HOME/.curlrc
-    echo 'retry = 5' >> $HOME/.curlrc
+    echo 'retry = 60' >> $HOME/.curlrc
+    echo 'retry-delay = 5' >> $HOME/.curlrc
     echo 'retry-all-errors' >> $HOME/.curlrc
+    # Don't clutter the logs with progress bars
+    echo 'no-progress-meter' >> $HOME/.curlrc
+    # Ensure errors cause the script to fail, but show the response body
+    echo 'fail-with-body' >> $HOME/.curlrc
 
     if [ -d /mnt/secrets/certs ]; then
       cp /mnt/secrets/certs/ca.pem /usr/share/pki/trust/anchors/private-ca.pem
@@ -76,7 +82,7 @@ data:
     -H 'Content-Type: application-json' \
     -d '{"vectors":{"size":4,"distance":"Dot"}}' \
     -H  "${API_KEY_HEADER}" \
-    $QDRANT_URL/collections/${QDRANT_COLLECTION} --fail-with-body
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}
 
     # Insert points
     curl -X PUT \
@@ -90,11 +96,11 @@ data:
       {"id":6,"vector":[0.35, 0.08, 0.11, 0.44],"payload":{"city":"Mumbai"}}
     ]}' \
     -H  "${API_KEY_HEADER}" \
-    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points --fail-with-body
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points
 
     # Run query
     curl -X POST \
     -H 'Content-Type: application-json' \
     -d '{"vector":[0.2, 0.1, 0.9, 0.7],"limit":3}' \
     -H  "${API_KEY_HEADER}" \
-    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points/search --fail-with-body
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points/search

test/integration/https.bats

@@ -21,4 +21,4 @@ setup_file() {
 @test "helm test - with https" {
     run helm test qdrant -n qdrant-helm-integration --logs
     [ $status -eq 0 ]
-}
+}

test/integration/setup_suite.bash

@@ -1,7 +1,12 @@
 setup_suite() {
     kind create cluster -n qdrant-helm-integration
     kubectl create serviceaccount default -n default
-    kubectl -n default run curl --image=docker.io/curlimages/curl --command -- sh -c "sleep 3600"
+    kubectl -n default run curl --image=docker.io/curlimages/curl --command -- sh -c '
+    echo "connect-timeout = 5" > $HOME/.curlrc;
+    echo "retry = 60" >> $HOME/.curlrc;
+    echo "retry-delay = 5" >> $HOME/.curlrc;
+    echo "retry-all-errors" >> $HOME/.curlrc;
+    sleep 3600'
     kubectl wait --for=condition=Ready pod/curl -n default --timeout=300s
     kubectl create namespace qdrant-helm-integration
     kubectl create serviceaccount default -n qdrant-helm-integration || true

test/integration/snapshot_persistence.bats

@@ -32,4 +32,5 @@ setup_file() {
 
 teardown_file() {
     helm delete qdrant -n qdrant-helm-integration || true
+    kubectl wait --for=delete pod/qdrant-0 -n qdrant-helm-integration --timeout=300s
 }