mirror of
https://github.com/cozystack/cozystack.git
synced 2026-03-18 04:48:54 +00:00
Compare commits
6 Commits
main
...
feat/sched
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7c59b6dc51 | ||
|
|
43d49b6e46 | ||
|
|
1024ee7607 | ||
|
|
6046a31e8c | ||
|
|
0387fae62c | ||
|
|
b821c0748e |
@@ -1,25 +0,0 @@
|
||||
<!--
|
||||
https://github.com/cozystack/cozystack/releases/tag/v1.1.2
|
||||
-->
|
||||
|
||||
## Fixes
|
||||
|
||||
* **[bucket] Fix S3 Manager endpoint mismatch with COSI credentials**: The S3 Manager UI previously constructed an `s3.<tenant>.<cluster-domain>` endpoint even though COSI-issued bucket credentials point to the root-level S3 endpoint. This caused login failures with "invalid credentials" despite valid secrets. The deployment now uses the actual endpoint from `BucketInfo`, with the old namespace-based endpoint kept only as a fallback before `BucketAccess` secrets exist ([**@IvanHunters**](https://github.com/IvanHunters) in #2211, #2215).
|
||||
|
||||
* **[platform] Fix spurious OpenAPI post-processing errors on cozystack-api startup**: The OpenAPI post-processor was being invoked for non-`apps.cozystack.io` group versions where the base `Application*` schemas do not exist, producing noisy startup errors on every API server launch. It now skips those non-apps group versions gracefully instead of returning an error ([**@kvaps**](https://github.com/kvaps) in #2212, #2217).
|
||||
|
||||
## Documentation
|
||||
|
||||
* **[website] Add troubleshooting for packages stuck in `DependenciesNotReady`**: Added an operations guide that explains how to diagnose missing package dependencies in operator logs and corrected the packages management development docs to use the current `make image-packages` target ([**@kvaps**](https://github.com/kvaps) in cozystack/website#450).
|
||||
|
||||
* **[website] Reorder installation docs to install the operator before the platform package**: Updated the platform installation guide and tutorial so the setup sequence consistently installs the Cozystack operator first, then prepares and applies the Platform Package, matching the rest of the documentation set ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#449).
|
||||
|
||||
* **[website] Add automated installation guide for the Ansible collection**: Added a full guide for deploying Cozystack with the `cozystack.installer` collection, including inventory examples, distro-specific playbooks, configuration reference, and explicit version pinning guidance ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#442).
|
||||
|
||||
* **[website] Expand monitoring and platform architecture reference docs**: Added a tenant custom metrics collection guide for `VMServiceScrape` and `VMPodScrape`, and documented `PackageSource`/`Package` architecture, reconciliation flow, rollback behavior, and the `cozypkg` workflow in Key Concepts ([**@IvanHunters**](https://github.com/IvanHunters) in cozystack/website#444, cozystack/website#445).
|
||||
|
||||
* **[website] Improve operations guides for CA rotation and Velero backups**: Completed the CA rotation documentation with dry-run and post-rotation credential retrieval steps, and expanded the backup configuration guide with concrete examples, verification commands, and clearer operator procedures ([**@kvaps**](https://github.com/kvaps) in cozystack/website#406; [**@androndo**](https://github.com/androndo) in cozystack/website#440).
|
||||
|
||||
---
|
||||
|
||||
**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.1.1...v1.1.2
|
||||
@@ -230,6 +230,10 @@ func applyListInputOverrides(schema map[string]any, kind string, openAPIProps ma
|
||||
kafkaProps["storageClass"] = storageClassListInput()
|
||||
zkProps := ensureSchemaPath(schema, "spec", "zookeeper")
|
||||
zkProps["storageClass"] = storageClassListInput()
|
||||
|
||||
case "Tenant":
|
||||
specProps := ensureSchemaPath(schema, "spec")
|
||||
specProps["schedulingClass"] = schedulingClassListInput()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,6 +250,20 @@ func storageClassListInput() map[string]any {
|
||||
}
|
||||
}
|
||||
|
||||
// schedulingClassListInput returns a listInput field config for a schedulingClass dropdown
|
||||
// backed by the cluster's available SchedulingClass CRs.
|
||||
func schedulingClassListInput() map[string]any {
|
||||
return map[string]any{
|
||||
"type": "listInput",
|
||||
"customProps": map[string]any{
|
||||
"valueUri": "/api/clusters/{cluster}/k8s/apis/cozystack.io/v1alpha1/schedulingclasses",
|
||||
"keysToValue": []any{"metadata", "name"},
|
||||
"keysToLabel": []any{"metadata", "name"},
|
||||
"allowEmpty": true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ensureArrayItemProps ensures that parentProps[fieldName].items.properties exists
|
||||
// and returns the items properties map. Used for overriding fields inside array items.
|
||||
func ensureArrayItemProps(parentProps map[string]any, fieldName string) map[string]any {
|
||||
|
||||
@@ -8,11 +8,14 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/cozystack/cozystack/pkg/lineage"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/client-go/dynamic"
|
||||
"k8s.io/client-go/rest"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
|
||||
@@ -31,6 +34,11 @@ const (
|
||||
ManagerGroupKey = "apps.cozystack.io/application.group"
|
||||
ManagerKindKey = "apps.cozystack.io/application.kind"
|
||||
ManagerNameKey = "apps.cozystack.io/application.name"
|
||||
|
||||
// Scheduling constants
|
||||
SchedulingClassLabel = "scheduling.cozystack.io/class"
|
||||
SchedulingClassAnnotation = "scheduler.cozystack.io/scheduling-class"
|
||||
CozystackSchedulerName = "cozystack-scheduler"
|
||||
)
|
||||
|
||||
// getResourceSelectors returns the appropriate ApplicationDefinitionResources for a given GroupKind
|
||||
@@ -115,6 +123,11 @@ func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Req
|
||||
|
||||
h.applyLabels(obj, labels)
|
||||
|
||||
if err := h.applySchedulingClass(ctx, obj, req.Namespace); err != nil {
|
||||
logger.Error(err, "error applying scheduling class")
|
||||
return admission.Errored(500, fmt.Errorf("error applying scheduling class: %w", err))
|
||||
}
|
||||
|
||||
mutated, err := json.Marshal(obj)
|
||||
if err != nil {
|
||||
return admission.Errored(500, fmt.Errorf("marshal mutated pod: %w", err))
|
||||
@@ -185,6 +198,57 @@ func (h *LineageControllerWebhook) applyLabels(o *unstructured.Unstructured, lab
|
||||
o.SetLabels(existing)
|
||||
}
|
||||
|
||||
// applySchedulingClass injects schedulerName and scheduling-class annotation
|
||||
// into Pods whose namespace carries the scheduling.cozystack.io/class label.
|
||||
// If the referenced SchedulingClass CR does not exist (e.g. the scheduler
|
||||
// package is not installed), the injection is silently skipped so that pods
|
||||
// are not left Pending.
|
||||
func (h *LineageControllerWebhook) applySchedulingClass(ctx context.Context, obj *unstructured.Unstructured, namespace string) error {
|
||||
if obj.GetKind() != "Pod" {
|
||||
return nil
|
||||
}
|
||||
|
||||
ns := &corev1.Namespace{}
|
||||
if err := h.Get(ctx, client.ObjectKey{Name: namespace}, ns); err != nil {
|
||||
return fmt.Errorf("getting namespace %s: %w", namespace, err)
|
||||
}
|
||||
|
||||
schedulingClass, ok := ns.Labels[SchedulingClassLabel]
|
||||
if !ok || schedulingClass == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify that the referenced SchedulingClass CR exists.
|
||||
// If the CRD is not installed or the CR is missing, skip injection
|
||||
// so that pods are not stuck Pending on a non-existent scheduler.
|
||||
_, err := h.dynClient.Resource(schedulingClassGVR).Get(ctx, schedulingClass, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger := log.FromContext(ctx)
|
||||
logger.Info("SchedulingClass not found, skipping scheduler injection",
|
||||
"schedulingClass", schedulingClass, "namespace", namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := unstructured.SetNestedField(obj.Object, CozystackSchedulerName, "spec", "schedulerName"); err != nil {
|
||||
return fmt.Errorf("setting schedulerName: %w", err)
|
||||
}
|
||||
|
||||
annotations := obj.GetAnnotations()
|
||||
if annotations == nil {
|
||||
annotations = make(map[string]string)
|
||||
}
|
||||
annotations[SchedulingClassAnnotation] = schedulingClass
|
||||
obj.SetAnnotations(annotations)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var schedulingClassGVR = schema.GroupVersionResource{
|
||||
Group: "cozystack.io",
|
||||
Version: "v1alpha1",
|
||||
Resource: "schedulingclasses",
|
||||
}
|
||||
|
||||
func (h *LineageControllerWebhook) decodeUnstructured(req admission.Request, out *unstructured.Unstructured) error {
|
||||
if h.decoder != nil {
|
||||
if err := h.decoder.Decode(req, out); err == nil {
|
||||
|
||||
@@ -317,7 +317,11 @@ func (w *WrappedControllerService) ControllerPublishVolume(ctx context.Context,
|
||||
"ownerReferences": []interface{}{vmiOwnerRef},
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"endpointSelector": buildEndpointSelector([]string{vmName}),
|
||||
"endpointSelector": map[string]interface{}{
|
||||
"matchLabels": map[string]interface{}{
|
||||
"kubevirt.io/vm": vmName,
|
||||
},
|
||||
},
|
||||
"egress": []interface{}{
|
||||
map[string]interface{}{
|
||||
"toEndpoints": []interface{}{
|
||||
@@ -437,13 +441,6 @@ func (w *WrappedControllerService) addCNPOwnerReference(ctx context.Context, nam
|
||||
if err := unstructured.SetNestedSlice(existing.Object, ownerRefs, "metadata", "ownerReferences"); err != nil {
|
||||
return status.Errorf(codes.Internal, "failed to set ownerReferences: %v", err)
|
||||
}
|
||||
|
||||
// Rebuild endpointSelector to include all VMs
|
||||
selector := buildEndpointSelector(vmNamesFromOwnerRefs(ownerRefs))
|
||||
if err := unstructured.SetNestedField(existing.Object, selector, "spec", "endpointSelector"); err != nil {
|
||||
return status.Errorf(codes.Internal, "failed to set endpointSelector: %v", err)
|
||||
}
|
||||
|
||||
if _, err := w.dynamicClient.Resource(ciliumNetworkPolicyGVR).Namespace(namespace).Update(ctx, existing, metav1.UpdateOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -489,13 +486,6 @@ func (w *WrappedControllerService) removeCNPOwnerReference(ctx context.Context,
|
||||
if err := unstructured.SetNestedSlice(existing.Object, remaining, "metadata", "ownerReferences"); err != nil {
|
||||
return status.Errorf(codes.Internal, "failed to set ownerReferences: %v", err)
|
||||
}
|
||||
|
||||
// Rebuild endpointSelector from remaining VMs
|
||||
selector := buildEndpointSelector(vmNamesFromOwnerRefs(remaining))
|
||||
if err := unstructured.SetNestedField(existing.Object, selector, "spec", "endpointSelector"); err != nil {
|
||||
return status.Errorf(codes.Internal, "failed to set endpointSelector: %v", err)
|
||||
}
|
||||
|
||||
if _, err := w.dynamicClient.Resource(ciliumNetworkPolicyGVR).Namespace(namespace).Update(ctx, existing, metav1.UpdateOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -504,37 +494,6 @@ func (w *WrappedControllerService) removeCNPOwnerReference(ctx context.Context,
|
||||
})
|
||||
}
|
||||
|
||||
// buildEndpointSelector returns an endpointSelector using matchExpressions
|
||||
// so that multiple VMs can be listed in a single selector.
|
||||
func buildEndpointSelector(vmNames []string) map[string]interface{} {
|
||||
values := make([]interface{}, len(vmNames))
|
||||
for i, name := range vmNames {
|
||||
values[i] = name
|
||||
}
|
||||
return map[string]interface{}{
|
||||
"matchExpressions": []interface{}{
|
||||
map[string]interface{}{
|
||||
"key": "kubevirt.io/vm",
|
||||
"operator": "In",
|
||||
"values": values,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// vmNamesFromOwnerRefs extracts VM names from ownerReferences.
|
||||
func vmNamesFromOwnerRefs(ownerRefs []interface{}) []string {
|
||||
var names []string
|
||||
for _, ref := range ownerRefs {
|
||||
if refMap, ok := ref.(map[string]interface{}); ok {
|
||||
if name, ok := refMap["name"].(string); ok {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func hasRWXAccessMode(pvc *corev1.PersistentVolumeClaim) bool {
|
||||
for _, mode := range pvc.Spec.AccessModes {
|
||||
if mode == corev1.ReadWriteMany {
|
||||
|
||||
@@ -74,14 +74,15 @@ tenant-u1
|
||||
|
||||
### Common parameters
|
||||
|
||||
| Name | Description | Type | Value |
|
||||
| ---------------- | -------------------------------------------------------------------------------------------------------------------------- | --------------------- | ------- |
|
||||
| `host` | The hostname used to access tenant services (defaults to using the tenant name as a subdomain for its parent tenant host). | `string` | `""` |
|
||||
| `etcd` | Deploy own Etcd cluster. | `bool` | `false` |
|
||||
| `monitoring` | Deploy own Monitoring Stack. | `bool` | `false` |
|
||||
| `ingress` | Deploy own Ingress Controller. | `bool` | `false` |
|
||||
| `seaweedfs` | Deploy own SeaweedFS. | `bool` | `false` |
|
||||
| `resourceQuotas` | Define resource quotas for the tenant. | `map[string]quantity` | `{}` |
|
||||
| Name | Description | Type | Value |
|
||||
| ----------------- | -------------------------------------------------------------------------------------------------------------------------- | --------------------- | ------- |
|
||||
| `host` | The hostname used to access tenant services (defaults to using the tenant name as a subdomain for its parent tenant host). | `string` | `""` |
|
||||
| `etcd` | Deploy own Etcd cluster. | `bool` | `false` |
|
||||
| `monitoring` | Deploy own Monitoring Stack. | `bool` | `false` |
|
||||
| `ingress` | Deploy own Ingress Controller. | `bool` | `false` |
|
||||
| `seaweedfs` | Deploy own SeaweedFS. | `bool` | `false` |
|
||||
| `schedulingClass` | The name of a SchedulingClass CR to apply scheduling constraints for this tenant's workloads. | `string` | `""` |
|
||||
| `resourceQuotas` | Define resource quotas for the tenant. | `map[string]quantity` | `{}` |
|
||||
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -38,6 +38,12 @@
|
||||
{{- if .Values.seaweedfs }}
|
||||
{{- $seaweedfs = $tenantName }}
|
||||
{{- end }}
|
||||
|
||||
{{/* SchedulingClass: inherited from parent, can be set if parent has none */}}
|
||||
{{- $schedulingClass := $parentNamespace.schedulingClass | default "" }}
|
||||
{{- if and (not $schedulingClass) .Values.schedulingClass }}
|
||||
{{- $schedulingClass = .Values.schedulingClass }}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
@@ -58,6 +64,9 @@ metadata:
|
||||
namespace.cozystack.io/monitoring: {{ $monitoring | quote }}
|
||||
namespace.cozystack.io/seaweedfs: {{ $seaweedfs | quote }}
|
||||
namespace.cozystack.io/host: {{ $computedHost | quote }}
|
||||
{{- with $schedulingClass }}
|
||||
scheduling.cozystack.io/class: {{ . | quote }}
|
||||
{{- end }}
|
||||
alpha.kubevirt.io/auto-memory-limits-ratio: "1.0"
|
||||
ownerReferences:
|
||||
- apiVersion: v1
|
||||
@@ -86,4 +95,7 @@ stringData:
|
||||
monitoring: {{ $monitoring | quote }}
|
||||
seaweedfs: {{ $seaweedfs | quote }}
|
||||
host: {{ $computedHost | quote }}
|
||||
{{- with $schedulingClass }}
|
||||
schedulingClass: {{ . | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -39,6 +39,11 @@
|
||||
"x-kubernetes-int-or-string": true
|
||||
}
|
||||
},
|
||||
"schedulingClass": {
|
||||
"description": "The name of a SchedulingClass CR to apply scheduling constraints for this tenant's workloads.",
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"seaweedfs": {
|
||||
"description": "Deploy own SeaweedFS.",
|
||||
"type": "boolean",
|
||||
|
||||
@@ -17,5 +17,8 @@ ingress: false
|
||||
## @param {bool} seaweedfs - Deploy own SeaweedFS.
|
||||
seaweedfs: false
|
||||
|
||||
## @param {string} [schedulingClass] - The name of a SchedulingClass CR to apply scheduling constraints for this tenant's workloads.
|
||||
schedulingClass: ""
|
||||
|
||||
## @param {map[string]quantity} resourceQuotas - Define resource quotas for the tenant.
|
||||
resourceQuotas: {}
|
||||
|
||||
@@ -18,5 +18,5 @@ spec:
|
||||
path: system/backupstrategy-controller
|
||||
install:
|
||||
privileged: true
|
||||
namespace: cozy-backup-controller
|
||||
namespace: cozy-backupstrategy-controller
|
||||
releaseName: backupstrategy-controller
|
||||
|
||||
@@ -26,7 +26,6 @@ stringData:
|
||||
oidc-enabled: {{ .Values.authentication.oidc.enabled | quote }}
|
||||
oidc-insecure-skip-verify: {{ .Values.authentication.oidc.insecureSkipVerify | quote }}
|
||||
extra-keycloak-redirect-uri-for-dashboard: {{ index .Values.authentication.oidc.keycloakExtraRedirectUri | quote }}
|
||||
keycloak-internal-url: {{ .Values.authentication.oidc.keycloakInternalUrl | quote }}
|
||||
expose-services: {{ .Values.publishing.exposedServices | join "," | quote }}
|
||||
expose-ingress: {{ .Values.publishing.ingressName | quote }}
|
||||
expose-external-ips: {{ .Values.publishing.externalIPs | join "," | quote }}
|
||||
|
||||
@@ -54,11 +54,6 @@ authentication:
|
||||
enabled: false
|
||||
insecureSkipVerify: false
|
||||
keycloakExtraRedirectUri: ""
|
||||
# Internal URL to access KeyCloak realm for backend-to-backend requests (bypasses external DNS).
|
||||
# When set, oauth2-proxy uses --skip-oidc-discovery and routes all backend calls (token, jwks,
|
||||
# userinfo, logout) through this URL while keeping browser redirects on the external URL.
|
||||
# Example: http://keycloak-http.cozy-keycloak.svc:8080/realms/cozy
|
||||
keycloakInternalUrl: ""
|
||||
# Pod scheduling configuration
|
||||
scheduling:
|
||||
globalAppTopologySpreadConstraints: ""
|
||||
|
||||
@@ -4,14 +4,9 @@ metadata:
|
||||
name: {{ .Release.Name }}-defrag
|
||||
spec:
|
||||
schedule: "0 * * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
startingDeadlineSeconds: 300
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
activeDeadlineSeconds: 1800
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
|
||||
@@ -178,7 +178,3 @@ vmagent:
|
||||
urls:
|
||||
- http://vminsert-shortterm:8480/insert/0/prometheus
|
||||
- http://vminsert-longterm:8480/insert/0/prometheus
|
||||
## inlineScrapeConfig: |
|
||||
## - job_name: "custom"
|
||||
## static_configs:
|
||||
## - targets: ["my-service:9090"]
|
||||
|
||||
@@ -14,10 +14,3 @@ rules:
|
||||
- apiGroups: ["backups.cozystack.io"]
|
||||
resources: ["backupjobs"]
|
||||
verbs: ["create", "get", "list", "watch"]
|
||||
# Leader election (--leader-elect)
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["create", "patch"]
|
||||
|
||||
@@ -30,10 +30,6 @@ rules:
|
||||
- apiGroups: ["velero.io"]
|
||||
resources: ["backups", "restores"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "patch"]
|
||||
# Events from Recorder.Event() calls
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["create", "patch"]
|
||||
# Leader election (--leader-elect)
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
{{- $host := index .Values._cluster "root-host" }}
|
||||
{{- $oidcEnabled := index .Values._cluster "oidc-enabled" }}
|
||||
{{- $oidcInsecureSkipVerify := index .Values._cluster "oidc-insecure-skip-verify" }}
|
||||
{{- $keycloakInternalUrl := index .Values._cluster "keycloak-internal-url" | default "" }}
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -56,16 +55,7 @@ spec:
|
||||
- --http-address=0.0.0.0:8000
|
||||
- --redirect-url=https://dashboard.{{ $host }}/oauth2/callback
|
||||
- --oidc-issuer-url=https://keycloak.{{ $host }}/realms/cozy
|
||||
{{- if $keycloakInternalUrl }}
|
||||
- --skip-oidc-discovery
|
||||
- --login-url=https://keycloak.{{ $host }}/realms/cozy/protocol/openid-connect/auth
|
||||
- --redeem-url={{ $keycloakInternalUrl }}/protocol/openid-connect/token
|
||||
- --oidc-jwks-url={{ $keycloakInternalUrl }}/protocol/openid-connect/certs
|
||||
- --validate-url={{ $keycloakInternalUrl }}/protocol/openid-connect/userinfo
|
||||
- --backend-logout-url={{ $keycloakInternalUrl }}/protocol/openid-connect/logout?id_token_hint={id_token}
|
||||
{{- else }}
|
||||
- --backend-logout-url=https://keycloak.{{ $host }}/realms/cozy/protocol/openid-connect/logout?id_token_hint={id_token}
|
||||
{{- end }}
|
||||
- --whitelist-domain=keycloak.{{ $host }}
|
||||
- --email-domain=*
|
||||
- --pass-access-token=true
|
||||
|
||||
@@ -3,7 +3,7 @@ kind: CDI
|
||||
metadata:
|
||||
name: cdi
|
||||
spec:
|
||||
cloneStrategyOverride: csi-clone
|
||||
cloneStrategyOverride: copy
|
||||
config:
|
||||
{{- with .Values.uploadProxyURL }}
|
||||
uploadProxyURLOverride: {{ quote . }}
|
||||
|
||||
@@ -1,358 +0,0 @@
|
||||
diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go
|
||||
index f544493..98e7fde 100644
|
||||
--- a/pkg/client/linstor.go
|
||||
+++ b/pkg/client/linstor.go
|
||||
@@ -181,7 +181,7 @@ func LogLevel(s string) func(*Linstor) error {
|
||||
func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus, error) {
|
||||
var vols []volume.VolumeStatus
|
||||
|
||||
- resourcesByName := make(map[string][]lapi.Resource)
|
||||
+ resourcesByName := make(map[string][]lapi.ResourceWithVolumes)
|
||||
|
||||
resDefs, err := s.client.ResourceDefinitions.GetAll(ctx, lapi.RDGetAllRequest{WithVolumeDefinitions: true})
|
||||
if err != nil {
|
||||
@@ -194,7 +194,7 @@ func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus,
|
||||
}
|
||||
|
||||
for i := range allResources {
|
||||
- resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i].Resource)
|
||||
+ resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i])
|
||||
}
|
||||
|
||||
for _, rd := range resDefs {
|
||||
@@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu
|
||||
return err
|
||||
}
|
||||
|
||||
+ if params.RelocateAfterClone {
|
||||
+ logger.Debug("relocate resources to optimal nodes")
|
||||
+
|
||||
+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil {
|
||||
+ logger.WithError(err).Warn("resource relocation failed, volume is still usable")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
logger.Debug("reconcile extra properties")
|
||||
|
||||
err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
|
||||
@@ -1280,6 +1288,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v
|
||||
return err
|
||||
}
|
||||
|
||||
+ if snapParams != nil && snapParams.RelocateAfterRestore {
|
||||
+ logger.Debug("relocate resources to optimal nodes")
|
||||
+
|
||||
+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil {
|
||||
+ logger.WithError(err).Warn("resource relocation failed, volume is still usable")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
logger.Debug("reconcile extra properties")
|
||||
|
||||
err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
|
||||
@@ -1470,9 +1486,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
|
||||
return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name)
|
||||
}
|
||||
|
||||
- // Optimize the node we use to restore. It should be one of the preferred nodes, or just the first with a snapshot
|
||||
- // if no preferred nodes match.
|
||||
- var selectedNode string
|
||||
+ // Collect available snapshot nodes, preferring those matching the topology hints.
|
||||
+ var preferred, available []string
|
||||
for _, snapNode := range snap.Nodes {
|
||||
if err := s.NodeAvailable(ctx, snapNode); err != nil {
|
||||
logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available")
|
||||
@@ -1480,13 +1495,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
|
||||
}
|
||||
|
||||
if slices.Contains(preferredNodes, snapNode) {
|
||||
- // We found a perfect candidate.
|
||||
- selectedNode = snapNode
|
||||
- break
|
||||
- } else if selectedNode == "" {
|
||||
- // Set a fallback if we have no candidate yet.
|
||||
- selectedNode = snapNode
|
||||
+ preferred = append(preferred, snapNode)
|
||||
}
|
||||
+
|
||||
+ available = append(available, snapNode)
|
||||
+ }
|
||||
+
|
||||
+ // Pick a random node from preferred candidates, falling back to any available node.
|
||||
+ // Randomization distributes restore load across snapshot nodes, preventing all
|
||||
+ // clones of the same source from concentrating on a single node.
|
||||
+ candidates := preferred
|
||||
+ if len(candidates) == 0 {
|
||||
+ candidates = available
|
||||
+ }
|
||||
+
|
||||
+ var selectedNode string
|
||||
+ if len(candidates) > 0 {
|
||||
+ selectedNode = candidates[rand.Intn(len(candidates))]
|
||||
}
|
||||
|
||||
if selectedNode == "" {
|
||||
@@ -1679,6 +1704,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In
|
||||
return nil
|
||||
}
|
||||
|
||||
+const propertyRelocationTriggered = "Aux/csi-relocation-triggered"
|
||||
+
|
||||
+// relocateResources migrates replicas from their current nodes to the nodes that
|
||||
+// LINSTOR's autoplacer considers optimal. This is a best-effort, fire-and-forget
|
||||
+// operation: migrate-disk API is asynchronous and the volume remains usable on its
|
||||
+// current nodes even if relocation fails.
|
||||
+func (s *Linstor) relocateResources(ctx context.Context, volID, rgName string) error {
|
||||
+ logger := s.log.WithFields(logrus.Fields{
|
||||
+ "volume": volID,
|
||||
+ "resourceGroup": rgName,
|
||||
+ })
|
||||
+
|
||||
+ // Check if relocation was already triggered (idempotency guard).
|
||||
+ rd, err := s.client.ResourceDefinitions.Get(ctx, volID)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("failed to get resource definition: %w", err)
|
||||
+ }
|
||||
+
|
||||
+ if rd.Props[propertyRelocationTriggered] != "" {
|
||||
+ logger.Debug("relocation already triggered, skipping")
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ // Get current diskful nodes.
|
||||
+ resources, err := s.client.Resources.GetAll(ctx, volID)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("failed to list resources: %w", err)
|
||||
+ }
|
||||
+
|
||||
+ currentNodes := util.DeployedDiskfullyNodes(resources)
|
||||
+
|
||||
+ // Query optimal placement from LINSTOR autoplacer.
|
||||
+ sizeInfo, err := s.client.ResourceGroups.QuerySizeInfo(ctx, rgName, lapi.QuerySizeInfoRequest{})
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("failed to query size info: %w", err)
|
||||
+ }
|
||||
+
|
||||
+ if sizeInfo.SpaceInfo == nil || len(sizeInfo.SpaceInfo.NextSpawnResult) == 0 {
|
||||
+ logger.Debug("no spawn result from query-size-info, skipping relocation")
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ // Build a map of optimal node -> storage pool.
|
||||
+ optimalPools := make(map[string]string, len(sizeInfo.SpaceInfo.NextSpawnResult))
|
||||
+ for _, r := range sizeInfo.SpaceInfo.NextSpawnResult {
|
||||
+ optimalPools[r.NodeName] = r.StorPoolName
|
||||
+ }
|
||||
+
|
||||
+ // Compute nodes to remove (current but not optimal) and nodes to add (optimal but not current).
|
||||
+ var nodesToRemove, nodesToAdd []string
|
||||
+
|
||||
+ for _, node := range currentNodes {
|
||||
+ if _, ok := optimalPools[node]; !ok {
|
||||
+ nodesToRemove = append(nodesToRemove, node)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ for _, r := range sizeInfo.SpaceInfo.NextSpawnResult {
|
||||
+ if !slices.Contains(currentNodes, r.NodeName) {
|
||||
+ nodesToAdd = append(nodesToAdd, r.NodeName)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ // Only migrate min(remove, add) pairs.
|
||||
+ pairs := min(len(nodesToRemove), len(nodesToAdd))
|
||||
+ if pairs == 0 {
|
||||
+ logger.Debug("no relocation needed, placement is already optimal")
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ logger.Infof("relocating %d replica(s): %v -> %v", pairs, nodesToRemove[:pairs], nodesToAdd[:pairs])
|
||||
+
|
||||
+ // Mark relocation as triggered before starting migrations.
|
||||
+ err = s.client.ResourceDefinitions.Modify(ctx, volID, lapi.GenericPropsModify{
|
||||
+ OverrideProps: map[string]string{propertyRelocationTriggered: "true"},
|
||||
+ })
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("failed to set relocation property: %w", err)
|
||||
+ }
|
||||
+
|
||||
+ // Migrate each pair sequentially (LINSTOR serializes at the RD level anyway).
|
||||
+ for i := range pairs {
|
||||
+ fromNode := nodesToRemove[i]
|
||||
+ toNode := nodesToAdd[i]
|
||||
+ storPool := optimalPools[toNode]
|
||||
+
|
||||
+ logger := logger.WithFields(logrus.Fields{"from": fromNode, "to": toNode, "storagePool": storPool})
|
||||
+
|
||||
+ logger.Info("creating diskless resource on target node")
|
||||
+
|
||||
+ err := s.client.Resources.MakeAvailable(ctx, volID, toNode, lapi.ResourceMakeAvailable{})
|
||||
+ if err != nil {
|
||||
+ logger.WithError(err).Warn("failed to create diskless on target, skipping this pair")
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ logger.Info("initiating migrate-disk")
|
||||
+
|
||||
+ err = s.client.Resources.Migrate(ctx, volID, fromNode, toNode, storPool)
|
||||
+ if err != nil {
|
||||
+ logger.WithError(err).Warn("migrate-disk failed, skipping this pair")
|
||||
+ continue
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
// FindSnapsByID searches the snapshot in the backend
|
||||
func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) {
|
||||
snapshotId, err := volume.ParseSnapshotId(id)
|
||||
@@ -2173,7 +2306,7 @@ func (s *Linstor) Status(ctx context.Context, volId string) ([]string, *csi.Volu
|
||||
"volume": volId,
|
||||
}).Debug("getting assignments")
|
||||
|
||||
- ress, err := s.client.Resources.GetAll(ctx, volId)
|
||||
+ ress, err := s.client.Resources.GetResourceView(ctx, &lapi.ListOpts{Resource: []string{volId}})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to list resources for '%s': %w", volId, err)
|
||||
}
|
||||
@@ -2261,13 +2394,20 @@ func GetSnapshotRemoteAndReadiness(snap *lapi.Snapshot) (string, bool, error) {
|
||||
return "", slices.Contains(snap.Flags, lapiconsts.FlagSuccessful), nil
|
||||
}
|
||||
|
||||
-func NodesAndConditionFromResources(ress []lapi.Resource) ([]string, *csi.VolumeCondition) {
|
||||
+func NodesAndConditionFromResources(ress []lapi.ResourceWithVolumes) ([]string, *csi.VolumeCondition) {
|
||||
var allNodes, abnormalNodes []string
|
||||
|
||||
for i := range ress {
|
||||
res := &ress[i]
|
||||
|
||||
- allNodes = append(allNodes, res.NodeName)
|
||||
+ // A resource is a CSI publish target if any of its volumes were created
|
||||
+ // by ControllerPublishVolume, identified by the temporary-diskless-attach property.
|
||||
+ if slices.ContainsFunc(res.Volumes, func(v lapi.Volume) bool {
|
||||
+ createdFor, ok := v.Props[linstor.PropertyCreatedFor]
|
||||
+ return ok && createdFor == linstor.CreatedForTemporaryDisklessAttach
|
||||
+ }) {
|
||||
+ allNodes = append(allNodes, res.NodeName)
|
||||
+ }
|
||||
|
||||
if res.State == nil {
|
||||
abnormalNodes = append(abnormalNodes, res.NodeName)
|
||||
diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go
|
||||
index 39acd95..aed18ab 100644
|
||||
--- a/pkg/volume/parameter.go
|
||||
+++ b/pkg/volume/parameter.go
|
||||
@@ -50,6 +50,7 @@ const (
|
||||
nfsservicename
|
||||
nfssquash
|
||||
nfsrecoveryvolumesize
|
||||
+ relocateafterclone
|
||||
)
|
||||
|
||||
// Parameters configuration for linstor volumes.
|
||||
@@ -118,6 +119,8 @@ type Parameters struct {
|
||||
// NfsRecoveryVolumeSize sets the volume size (in bytes) of the recovery volume used by the NFS server.
|
||||
// Defaults to 300MiB.
|
||||
NfsRecoveryVolumeBytes int64
|
||||
+ // RelocateAfterClone triggers asynchronous relocation of replicas to optimal nodes after cloning.
|
||||
+ RelocateAfterClone bool
|
||||
}
|
||||
|
||||
const DefaultDisklessStoragePoolName = "DfltDisklessStorPool"
|
||||
@@ -140,6 +143,7 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters,
|
||||
NfsServiceName: "linstor-csi-nfs",
|
||||
NfsSquash: "no_root_squash",
|
||||
NfsRecoveryVolumeBytes: 300 * 1024 * 1024,
|
||||
+ RelocateAfterClone: true,
|
||||
}
|
||||
|
||||
for k, v := range params {
|
||||
@@ -287,6 +291,13 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters,
|
||||
}
|
||||
|
||||
p.NfsRecoveryVolumeBytes = s.Value()
|
||||
+ case relocateafterclone:
|
||||
+ val, err := strconv.ParseBool(v)
|
||||
+ if err != nil {
|
||||
+ return p, err
|
||||
+ }
|
||||
+
|
||||
+ p.RelocateAfterClone = val
|
||||
}
|
||||
}
|
||||
|
||||
diff --git a/pkg/volume/paramkey_enumer.go b/pkg/volume/paramkey_enumer.go
|
||||
index 5474963..2eb8a7c 100644
|
||||
--- a/pkg/volume/paramkey_enumer.go
|
||||
+++ b/pkg/volume/paramkey_enumer.go
|
||||
@@ -7,11 +7,11 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
-const _paramKeyName = "allowremotevolumeaccessautoplaceclientlistdisklessonremainingdisklessstoragepooldonotplacewithregexencryptionfsoptslayerlistmountoptsnodelistplacementcountplacementpolicyreplicasondifferentreplicasonsamesizekibstoragepoolpostmountxfsoptsresourcegroupusepvcnameoverprovisionxreplicasondifferentnfsconfigtemplatenfsservicenamenfssquashnfsrecoveryvolumesize"
|
||||
+const _paramKeyName = "allowremotevolumeaccessautoplaceclientlistdisklessonremainingdisklessstoragepooldonotplacewithregexencryptionfsoptslayerlistmountoptsnodelistplacementcountplacementpolicyreplicasondifferentreplicasonsamesizekibstoragepoolpostmountxfsoptsresourcegroupusepvcnameoverprovisionxreplicasondifferentnfsconfigtemplatenfsservicenamenfssquashnfsrecoveryvolumesizerelocateafterclone"
|
||||
|
||||
-var _paramKeyIndex = [...]uint16{0, 23, 32, 42, 61, 80, 99, 109, 115, 124, 133, 141, 155, 170, 189, 203, 210, 221, 237, 250, 260, 273, 293, 310, 324, 333, 354}
|
||||
+var _paramKeyIndex = [...]uint16{0, 23, 32, 42, 61, 80, 99, 109, 115, 124, 133, 141, 155, 170, 189, 203, 210, 221, 237, 250, 260, 273, 293, 310, 324, 333, 354, 372}
|
||||
|
||||
-const _paramKeyLowerName = "allowremotevolumeaccessautoplaceclientlistdisklessonremainingdisklessstoragepooldonotplacewithregexencryptionfsoptslayerlistmountoptsnodelistplacementcountplacementpolicyreplicasondifferentreplicasonsamesizekibstoragepoolpostmountxfsoptsresourcegroupusepvcnameoverprovisionxreplicasondifferentnfsconfigtemplatenfsservicenamenfssquashnfsrecoveryvolumesize"
|
||||
+const _paramKeyLowerName = "allowremotevolumeaccessautoplaceclientlistdisklessonremainingdisklessstoragepooldonotplacewithregexencryptionfsoptslayerlistmountoptsnodelistplacementcountplacementpolicyreplicasondifferentreplicasonsamesizekibstoragepoolpostmountxfsoptsresourcegroupusepvcnameoverprovisionxreplicasondifferentnfsconfigtemplatenfsservicenamenfssquashnfsrecoveryvolumesizerelocateafterclone"
|
||||
|
||||
func (i paramKey) String() string {
|
||||
if i < 0 || i >= paramKey(len(_paramKeyIndex)-1) {
|
||||
@@ -50,9 +50,10 @@ func _paramKeyNoOp() {
|
||||
_ = x[nfsservicename-(23)]
|
||||
_ = x[nfssquash-(24)]
|
||||
_ = x[nfsrecoveryvolumesize-(25)]
|
||||
+ _ = x[relocateafterclone-(26)]
|
||||
}
|
||||
|
||||
-var _paramKeyValues = []paramKey{allowremotevolumeaccess, autoplace, clientlist, disklessonremaining, disklessstoragepool, donotplacewithregex, encryption, fsopts, layerlist, mountopts, nodelist, placementcount, placementpolicy, replicasondifferent, replicasonsame, sizekib, storagepool, postmountxfsopts, resourcegroup, usepvcname, overprovision, xreplicasondifferent, nfsconfigtemplate, nfsservicename, nfssquash, nfsrecoveryvolumesize}
|
||||
+var _paramKeyValues = []paramKey{allowremotevolumeaccess, autoplace, clientlist, disklessonremaining, disklessstoragepool, donotplacewithregex, encryption, fsopts, layerlist, mountopts, nodelist, placementcount, placementpolicy, replicasondifferent, replicasonsame, sizekib, storagepool, postmountxfsopts, resourcegroup, usepvcname, overprovision, xreplicasondifferent, nfsconfigtemplate, nfsservicename, nfssquash, nfsrecoveryvolumesize, relocateafterclone}
|
||||
|
||||
var _paramKeyNameToValueMap = map[string]paramKey{
|
||||
_paramKeyName[0:23]: allowremotevolumeaccess,
|
||||
@@ -107,6 +108,8 @@ var _paramKeyNameToValueMap = map[string]paramKey{
|
||||
_paramKeyLowerName[324:333]: nfssquash,
|
||||
_paramKeyName[333:354]: nfsrecoveryvolumesize,
|
||||
_paramKeyLowerName[333:354]: nfsrecoveryvolumesize,
|
||||
+ _paramKeyName[354:372]: relocateafterclone,
|
||||
+ _paramKeyLowerName[354:372]: relocateafterclone,
|
||||
}
|
||||
|
||||
var _paramKeyNames = []string{
|
||||
@@ -136,6 +139,7 @@ var _paramKeyNames = []string{
|
||||
_paramKeyName[310:324],
|
||||
_paramKeyName[324:333],
|
||||
_paramKeyName[333:354],
|
||||
+ _paramKeyName[354:372],
|
||||
}
|
||||
|
||||
// paramKeyString retrieves an enum value from the enum constants string name.
|
||||
diff --git a/pkg/volume/snapshot_params.go b/pkg/volume/snapshot_params.go
|
||||
index d167cb8..50b70fb 100644
|
||||
--- a/pkg/volume/snapshot_params.go
|
||||
+++ b/pkg/volume/snapshot_params.go
|
||||
@@ -35,6 +35,7 @@ type SnapshotParameters struct {
|
||||
LinstorTargetUrl string `json:"linstor-target-url,omitempty"`
|
||||
LinstorTargetClusterID string `json:"linstor-target-cluster-id,omitempty"`
|
||||
LinstorTargetStoragePool string `json:"linstor-target-storage-pool,omitempty"`
|
||||
+ RelocateAfterRestore bool `json:"relocate-after-restore,omitempty"`
|
||||
}
|
||||
|
||||
func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParameters, error) {
|
||||
@@ -91,6 +92,13 @@ func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParamete
|
||||
p.LinstorTargetClusterID = v
|
||||
case "/linstor-target-storage-pool":
|
||||
p.LinstorTargetStoragePool = v
|
||||
+ case "/relocateAfterRestore":
|
||||
+ b, err := strconv.ParseBool(v)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+
|
||||
+ p.RelocateAfterRestore = b
|
||||
default:
|
||||
log.WithField("key", k).Warn("ignoring unknown snapshot parameter key")
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
diff --git a/cmd/linstor-csi/linstor-csi.go b/cmd/linstor-csi/linstor-csi.go
|
||||
index 143f6cee..bd28e06e 100644
|
||||
--- a/cmd/linstor-csi/linstor-csi.go
|
||||
+++ b/cmd/linstor-csi/linstor-csi.go
|
||||
@@ -41,22 +41,23 @@ import (
|
||||
|
||||
func main() {
|
||||
var (
|
||||
- lsEndpoint = flag.String("linstor-endpoint", "", "Controller API endpoint for LINSTOR")
|
||||
- lsSkipTLSVerification = flag.Bool("linstor-skip-tls-verification", false, "If true, do not verify tls")
|
||||
- csiEndpoint = flag.String("csi-endpoint", "unix:///var/lib/kubelet/plugins/linstor.csi.linbit.com/csi.sock", "CSI endpoint")
|
||||
- node = flag.String("node", "", "Node ID to pass to node service")
|
||||
- logLevel = flag.String("log-level", "info", "Enable debug log output. Choose from: panic, fatal, error, warn, info, debug")
|
||||
- rps = flag.Float64("linstor-api-requests-per-second", 0, "Maximum allowed number of LINSTOR API requests per second. Default: Unlimited")
|
||||
- burst = flag.Int("linstor-api-burst", 1, "Maximum number of API requests allowed before being limited by requests-per-second. Default: 1 (no bursting)")
|
||||
- bearerTokenFile = flag.String("bearer-token", "", "Read the bearer token from the given file and use it for authentication.")
|
||||
- propNs = flag.String("property-namespace", linstor.NamespcAuxiliary, "Limit the reported topology keys to properties from the given namespace.")
|
||||
- labelBySP = flag.Bool("label-by-storage-pool", true, "Set to false to disable labeling of nodes based on their configured storage pools.")
|
||||
- nodeCacheTimeout = flag.Duration("node-cache-timeout", 1*time.Minute, "Duration for which the results of node and storage pool related API responses should be cached.")
|
||||
- resourceCacheTimeout = flag.Duration("resource-cache-timeout", 30*time.Second, "Duration for which the results of resource related API responses should be cached.")
|
||||
- resyncAfter = flag.Duration("resync-after", 5*time.Minute, "Duration after which reconciliations (such as for VolumeSnapshotClasses) should be rerun. Set to 0 to disable.")
|
||||
- enableRWX = flag.Bool("enable-rwx", false, "Enable RWX support via NFS (requires running in Kubernetes).")
|
||||
- namespace = flag.String("nfs-service-namespace", "", "The namespace the NFS service is running in.")
|
||||
- reactorConfigMapName = flag.String("nfs-reactor-config-map-name", "linstor-csi-nfs-reactor-config", "Name of the config map used to store promoter configuration")
|
||||
+ lsEndpoint = flag.String("linstor-endpoint", "", "Controller API endpoint for LINSTOR")
|
||||
+ lsSkipTLSVerification = flag.Bool("linstor-skip-tls-verification", false, "If true, do not verify tls")
|
||||
+ csiEndpoint = flag.String("csi-endpoint", "unix:///var/lib/kubelet/plugins/linstor.csi.linbit.com/csi.sock", "CSI endpoint")
|
||||
+ node = flag.String("node", "", "Node ID to pass to node service")
|
||||
+ logLevel = flag.String("log-level", "info", "Enable debug log output. Choose from: panic, fatal, error, warn, info, debug")
|
||||
+ rps = flag.Float64("linstor-api-requests-per-second", 0, "Maximum allowed number of LINSTOR API requests per second. Default: Unlimited")
|
||||
+ burst = flag.Int("linstor-api-burst", 1, "Maximum number of API requests allowed before being limited by requests-per-second. Default: 1 (no bursting)")
|
||||
+ bearerTokenFile = flag.String("bearer-token", "", "Read the bearer token from the given file and use it for authentication.")
|
||||
+ propNs = flag.String("property-namespace", linstor.NamespcAuxiliary, "Limit the reported topology keys to properties from the given namespace.")
|
||||
+ labelBySP = flag.Bool("label-by-storage-pool", true, "Set to false to disable labeling of nodes based on their configured storage pools.")
|
||||
+ nodeCacheTimeout = flag.Duration("node-cache-timeout", 1*time.Minute, "Duration for which the results of node and storage pool related API responses should be cached.")
|
||||
+ resourceCacheTimeout = flag.Duration("resource-cache-timeout", 30*time.Second, "Duration for which the results of resource related API responses should be cached.")
|
||||
+ resyncAfter = flag.Duration("resync-after", 5*time.Minute, "Duration after which reconciliations (such as for VolumeSnapshotClasses) should be rerun. Set to 0 to disable.")
|
||||
+ enableRWX = flag.Bool("enable-rwx", false, "Enable RWX support via NFS (requires running in Kubernetes).")
|
||||
+ namespace = flag.String("nfs-service-namespace", "", "The namespace the NFS service is running in.")
|
||||
+ reactorConfigMapName = flag.String("nfs-reactor-config-map-name", "linstor-csi-nfs-reactor-config", "Name of the config map used to store promoter configuration")
|
||||
+ disableRWXBlockValidation = flag.Bool("disable-rwx-block-validation", false, "Disable KubeVirt VM ownership validation for RWX block volumes.")
|
||||
)
|
||||
|
||||
flag.Var(&volume.DefaultRemoteAccessPolicy, "default-remote-access-policy", "")
|
||||
@@ -169,6 +170,10 @@ func main() {
|
||||
opts = append(opts, driver.ConfigureRWX(*namespace, *reactorConfigMapName))
|
||||
}
|
||||
|
||||
+ if *disableRWXBlockValidation {
|
||||
+ opts = append(opts, driver.DisableRWXBlockValidation())
|
||||
+ }
|
||||
+
|
||||
drv, err := driver.NewDriver(opts...)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go
|
||||
index bea69a8b..a39674b6 100644
|
||||
--- a/pkg/driver/driver.go
|
||||
+++ b/pkg/driver/driver.go
|
||||
@@ -83,6 +83,8 @@ type Driver struct {
|
||||
topologyPrefix string
|
||||
// resyncAfter is the interval after which reconciliations should be retried
|
||||
resyncAfter time.Duration
|
||||
+ // disableRWXBlockValidation disables KubeVirt VM ownership validation for RWX block volumes
|
||||
+ disableRWXBlockValidation bool
|
||||
|
||||
// Embed for forward compatibility.
|
||||
csi.UnimplementedIdentityServer
|
||||
@@ -300,6 +302,17 @@ func ResyncAfter(resyncAfter time.Duration) func(*Driver) error {
|
||||
}
|
||||
}
|
||||
|
||||
+// DisableRWXBlockValidation disables the KubeVirt VM ownership validation for RWX block volumes.
|
||||
+// When disabled, the driver will not check if multiple pods using the same RWX block volume
|
||||
+// belong to the same VM. This may be needed in environments where the validation causes issues
|
||||
+// or when using RWX block volumes outside of KubeVirt.
|
||||
+func DisableRWXBlockValidation() func(*Driver) error {
|
||||
+ return func(d *Driver) error {
|
||||
+ d.disableRWXBlockValidation = true
|
||||
+ return nil
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
// GetPluginInfo https://github.com/container-storage-interface/spec/blob/v1.9.0/spec.md#getplugininfo
|
||||
func (d Driver) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) {
|
||||
return &csi.GetPluginInfoResponse{
|
||||
@@ -751,6 +764,14 @@ func (d Driver) ControllerPublishVolume(ctx context.Context, req *csi.Controller
|
||||
// ReadWriteMany block volume
|
||||
rwxBlock := req.VolumeCapability.AccessMode.GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER && req.VolumeCapability.GetBlock() != nil
|
||||
|
||||
+ // Validate RWX block attachment to prevent misuse of allow-two-primaries
|
||||
+ if rwxBlock && !d.disableRWXBlockValidation {
|
||||
+ if _, err := utils.ValidateRWXBlockAttachment(ctx, d.kubeClient, d.log, req.GetVolumeId()); err != nil {
|
||||
+ return nil, status.Errorf(codes.FailedPrecondition,
|
||||
+ "ControllerPublishVolume failed for %s: %v", req.GetVolumeId(), err)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
devPath, err := d.Assignments.Attach(ctx, req.GetVolumeId(), req.GetNodeId(), rwxBlock)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal,
|
||||
diff --git a/pkg/utils/rwx_validation.go b/pkg/utils/rwx_validation.go
|
||||
new file mode 100644
|
||||
index 00000000..9fe82768
|
||||
--- /dev/null
|
||||
+++ b/pkg/utils/rwx_validation.go
|
||||
@@ -0,0 +1,263 @@
|
||||
+/*
|
||||
+CSI Driver for Linstor
|
||||
+Copyright © 2018 LINBIT USA, LLC
|
||||
+
|
||||
+This program is free software; you can redistribute it and/or modify
|
||||
+it under the terms of the GNU General Public License as published by
|
||||
+the Free Software Foundation; either version 2 of the License, or
|
||||
+(at your option) any later version.
|
||||
+
|
||||
+This program is distributed in the hope that it will be useful,
|
||||
+but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+GNU General Public License for more details.
|
||||
+
|
||||
+You should have received a copy of the GNU General Public License
|
||||
+along with this program; if not, see <http://www.gnu.org/licenses/>.
|
||||
+*/
|
||||
+
|
||||
+package utils
|
||||
+
|
||||
+import (
|
||||
+ "context"
|
||||
+ "fmt"
|
||||
+
|
||||
+ "github.com/sirupsen/logrus"
|
||||
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
+ "k8s.io/apimachinery/pkg/runtime/schema"
|
||||
+ "k8s.io/client-go/dynamic"
|
||||
+)
|
||||
+
|
||||
+// KubeVirtVMLabel is the label that KubeVirt adds to pods to identify the VM they belong to.
|
||||
+const KubeVirtVMLabel = "vm.kubevirt.io/name"
|
||||
+
|
||||
+// KubeVirtHotplugDiskLabel is the label that KubeVirt adds to hotplug disk pods.
|
||||
+const KubeVirtHotplugDiskLabel = "kubevirt.io"
|
||||
+
|
||||
+// PodGVR is the GroupVersionResource for pods.
|
||||
+var PodGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
|
||||
+
|
||||
+// PVGVR is the GroupVersionResource for persistent volumes.
|
||||
+var PVGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "persistentvolumes"}
|
||||
+
|
||||
+// ValidateRWXBlockAttachment checks that RWX block volumes are only used by pods belonging to the same VM.
|
||||
+// This prevents misuse of allow-two-primaries while still permitting live migration.
|
||||
+// Returns the VM name if validation passes, or an error if:
|
||||
+// - Multiple pods from different VMs are trying to use the same volume
|
||||
+// - A pod without the KubeVirt VM label is trying to use a volume already attached elsewhere (strict mode)
|
||||
+// Returns empty string for VM name when no pods are using the volume or validation is skipped.
|
||||
+func ValidateRWXBlockAttachment(ctx context.Context, kubeClient dynamic.Interface, log *logrus.Entry, volumeID string) (string, error) {
|
||||
+ log.WithField("volumeID", volumeID).Info("validateRWXBlockAttachment called")
|
||||
+
|
||||
+ if kubeClient == nil {
|
||||
+ // Not running in Kubernetes, skip validation
|
||||
+ log.Warn("validateRWXBlockAttachment: kubeClient is nil, skipping validation")
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Get PV to find PVC reference
|
||||
+ pv, err := kubeClient.Resource(PVGVR).Get(ctx, volumeID, metav1.GetOptions{})
|
||||
+ if err != nil {
|
||||
+ log.WithError(err).Warn("cannot validate RWX attachment: failed to get PV")
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Verify that PV's volumeHandle matches the volumeID
|
||||
+ volumeHandle, found, err := unstructured.NestedString(pv.Object, "spec", "csi", "volumeHandle")
|
||||
+ if err != nil {
|
||||
+ log.WithError(err).Warnf("cannot validate RWX attachment: failed to read volumeHandle for PV %s", volumeID)
|
||||
+
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ if !found {
|
||||
+ log.Warnf("cannot validate RWX attachment: volumeHandle not found for PV %s", volumeID)
|
||||
+
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ if volumeHandle != volumeID {
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "volumeID": volumeID,
|
||||
+ "volumeHandle": volumeHandle,
|
||||
+ }).Warn("cannot validate RWX attachment: PV volumeHandle does not match volumeID")
|
||||
+
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Extract claimRef from PV
|
||||
+ claimRef, found, _ := unstructured.NestedMap(pv.Object, "spec", "claimRef")
|
||||
+ if !found {
|
||||
+ log.Warn("cannot validate RWX attachment: PV has no claimRef")
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ pvcName, _, _ := unstructured.NestedString(claimRef, "name")
|
||||
+ pvcNamespace, _, _ := unstructured.NestedString(claimRef, "namespace")
|
||||
+
|
||||
+ if pvcNamespace == "" || pvcName == "" {
|
||||
+ log.Warn("cannot validate RWX attachment: PVC name or namespace is empty in claimRef")
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // List all pods in the namespace
|
||||
+ podList, err := kubeClient.Resource(PodGVR).Namespace(pvcNamespace).List(ctx, metav1.ListOptions{})
|
||||
+ if err != nil {
|
||||
+ return "", fmt.Errorf("failed to list pods in namespace %s: %w", pvcNamespace, err)
|
||||
+ }
|
||||
+
|
||||
+ // Filter pods that use this PVC and are in a running/pending state
|
||||
+ type podInfo struct {
|
||||
+ name string
|
||||
+ vmName string
|
||||
+ }
|
||||
+
|
||||
+ var podsUsingPVC []podInfo
|
||||
+
|
||||
+ for _, item := range podList.Items {
|
||||
+ // Get pod phase from status
|
||||
+ phase, _, _ := unstructured.NestedString(item.Object, "status", "phase")
|
||||
+ if phase == "Succeeded" || phase == "Failed" {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ // Check if pod uses the PVC
|
||||
+ volumes, found, _ := unstructured.NestedSlice(item.Object, "spec", "volumes")
|
||||
+ if !found {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ for _, vol := range volumes {
|
||||
+ volMap, ok := vol.(map[string]interface{})
|
||||
+ if !ok {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ claimName, found, _ := unstructured.NestedString(volMap, "persistentVolumeClaim", "claimName")
|
||||
+ if !found || claimName != pvcName {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ // Extract VM name, handling both regular and hotplug disk pods
|
||||
+ vmName, err := GetVMNameFromPod(ctx, kubeClient, log, &item)
|
||||
+ if err != nil {
|
||||
+ log.WithError(err).WithField("pod", item.GetName()).Warn("failed to get VM name from pod")
|
||||
+ // Continue with empty vmName - will be caught by strict mode check
|
||||
+ vmName = ""
|
||||
+ }
|
||||
+
|
||||
+ podsUsingPVC = append(podsUsingPVC, podInfo{
|
||||
+ name: item.GetName(),
|
||||
+ vmName: vmName,
|
||||
+ })
|
||||
+
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ // If 0 or 1 pod uses the PVC, no conflict possible
|
||||
+ if len(podsUsingPVC) <= 1 {
|
||||
+ // Return VM name if there's exactly one pod
|
||||
+ if len(podsUsingPVC) == 1 {
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "volumeID": volumeID,
|
||||
+ "vmName": podsUsingPVC[0].vmName,
|
||||
+ "podCount": 1,
|
||||
+ "pvcNamespace": pvcNamespace,
|
||||
+ "pvcName": pvcName,
|
||||
+ }).Info("validateRWXBlockAttachment: single pod found, returning VM name")
|
||||
+
|
||||
+ return podsUsingPVC[0].vmName, nil
|
||||
+ }
|
||||
+
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "volumeID": volumeID,
|
||||
+ "pvcNamespace": pvcNamespace,
|
||||
+ "pvcName": pvcName,
|
||||
+ }).Info("validateRWXBlockAttachment: no pods found using PVC")
|
||||
+
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Check that all pods belong to the same VM
|
||||
+ var vmName string
|
||||
+ for _, pod := range podsUsingPVC {
|
||||
+ if pod.vmName == "" {
|
||||
+ // Strict mode: if any pod doesn't have the KubeVirt label and there are multiple pods,
|
||||
+ // deny the attachment
|
||||
+ return "", fmt.Errorf("RWX block volume %s/%s is used by multiple pods but pod %s does not have the %s label; "+
|
||||
+ "RWX block volumes with allow-two-primaries are only supported for KubeVirt live migration",
|
||||
+ pvcNamespace, pvcName, pod.name, KubeVirtVMLabel)
|
||||
+ }
|
||||
+
|
||||
+ if vmName == "" {
|
||||
+ vmName = pod.vmName
|
||||
+ } else if vmName != pod.vmName {
|
||||
+ // Different VMs are trying to use the same volume
|
||||
+ return "", fmt.Errorf("RWX block volume %s/%s is being used by pods from different VMs (%s and %s); "+
|
||||
+ "this is not supported - RWX block volumes with allow-two-primaries are only for live migration of a single VM",
|
||||
+ pvcNamespace, pvcName, vmName, pod.vmName)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "pvcNamespace": pvcNamespace,
|
||||
+ "pvcName": pvcName,
|
||||
+ "vmName": vmName,
|
||||
+ "podCount": len(podsUsingPVC),
|
||||
+ }).Debug("RWX block attachment validated: all pods belong to the same VM (likely live migration)")
|
||||
+
|
||||
+ return vmName, nil
|
||||
+}
|
||||
+
|
||||
+// GetVMNameFromPod extracts the VM name from a pod, handling both regular virt-launcher pods
|
||||
+// and hotplug disk pods (which reference the virt-launcher pod via ownerReferences).
|
||||
+func GetVMNameFromPod(ctx context.Context, kubeClient dynamic.Interface, log *logrus.Entry, pod *unstructured.Unstructured) (string, error) {
|
||||
+ labels := pod.GetLabels()
|
||||
+ if labels == nil {
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Direct case: pod has vm.kubevirt.io/name label (virt-launcher pod)
|
||||
+ if vmName, ok := labels[KubeVirtVMLabel]; ok && vmName != "" {
|
||||
+ return vmName, nil
|
||||
+ }
|
||||
+
|
||||
+ // Hotplug disk case: pod has kubevirt.io: hotplug-disk label
|
||||
+ // Follow ownerReferences to find the virt-launcher pod
|
||||
+ if hotplugValue, ok := labels[KubeVirtHotplugDiskLabel]; ok && hotplugValue == "hotplug-disk" {
|
||||
+ ownerRefs := pod.GetOwnerReferences()
|
||||
+ for _, owner := range ownerRefs {
|
||||
+ if owner.Kind != "Pod" || owner.Controller == nil || !*owner.Controller {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ // Get the owner pod (virt-launcher)
|
||||
+ ownerPod, err := kubeClient.Resource(PodGVR).Namespace(pod.GetNamespace()).Get(ctx, owner.Name, metav1.GetOptions{})
|
||||
+ if err != nil {
|
||||
+ return "", fmt.Errorf("failed to get owner pod %s: %w", owner.Name, err)
|
||||
+ }
|
||||
+
|
||||
+ // Extract VM name from owner pod
|
||||
+ ownerLabels := ownerPod.GetLabels()
|
||||
+ if ownerLabels != nil {
|
||||
+ if vmName, ok := ownerLabels[KubeVirtVMLabel]; ok && vmName != "" {
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "hotplugPod": pod.GetName(),
|
||||
+ "virtLauncher": owner.Name,
|
||||
+ "vmName": vmName,
|
||||
+ }).Debug("resolved VM name from hotplug disk pod via owner reference")
|
||||
+
|
||||
+ return vmName, nil
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return "", fmt.Errorf("owner pod %s does not have %s label", owner.Name, KubeVirtVMLabel)
|
||||
+ }
|
||||
+
|
||||
+ return "", fmt.Errorf("hotplug disk pod %s has no controller owner reference", pod.GetName())
|
||||
+ }
|
||||
+
|
||||
+ return "", nil
|
||||
+}
|
||||
diff --git a/pkg/utils/rwx_validation_test.go b/pkg/utils/rwx_validation_test.go
|
||||
new file mode 100644
|
||||
index 00000000..d75690f9
|
||||
--- /dev/null
|
||||
+++ b/pkg/utils/rwx_validation_test.go
|
||||
@@ -0,0 +1,342 @@
|
||||
+/*
|
||||
+CSI Driver for Linstor
|
||||
+Copyright © 2018 LINBIT USA, LLC
|
||||
+
|
||||
+This program is free software; you can redistribute it and/or modify
|
||||
+it under the terms of the GNU General Public License as published by
|
||||
+the Free Software Foundation; either version 2 of the License, or
|
||||
+(at your option) any later version.
|
||||
+
|
||||
+This program is distributed in the hope that it will be useful,
|
||||
+but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+GNU General Public License for more details.
|
||||
+
|
||||
+You should have received a copy of the GNU General Public License
|
||||
+along with this program; if not, see <http://www.gnu.org/licenses/>.
|
||||
+*/
|
||||
+
|
||||
+package utils
|
||||
+
|
||||
+import (
|
||||
+ "context"
|
||||
+ "testing"
|
||||
+
|
||||
+ "github.com/sirupsen/logrus"
|
||||
+ "github.com/stretchr/testify/assert"
|
||||
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
+ "k8s.io/apimachinery/pkg/runtime"
|
||||
+ "k8s.io/apimachinery/pkg/runtime/schema"
|
||||
+ dynamicfake "k8s.io/client-go/dynamic/fake"
|
||||
+)
|
||||
+
|
||||
+func TestValidateRWXBlockAttachment(t *testing.T) {
|
||||
+ testCases := []struct {
|
||||
+ name string
|
||||
+ pods []*unstructured.Unstructured
|
||||
+ pvcName string
|
||||
+ namespace string
|
||||
+ expectError bool
|
||||
+ errorMsg string
|
||||
+ }{
|
||||
+ {
|
||||
+ name: "no pods using PVC",
|
||||
+ pods: []*unstructured.Unstructured{},
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "single pod using PVC",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "two pods same VM (live migration)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-abc", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm1-xyz", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "two pods different VMs (should fail)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-abc", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm2-xyz", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: true,
|
||||
+ errorMsg: "different VMs",
|
||||
+ },
|
||||
+ {
|
||||
+ name: "pod without KubeVirt label when multiple pods exist (strict mode)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "default", "test-pvc", map[string]string{}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: true,
|
||||
+ errorMsg: "does not have the vm.kubevirt.io/name label",
|
||||
+ },
|
||||
+ {
|
||||
+ name: "completed pods should be ignored",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Succeeded"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "failed pods should be ignored",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Failed"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "pods in different namespace should not conflict",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "other", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "pods using different PVCs should not conflict",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "default", "other-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "three pods from same VM (multi-node live migration scenario)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-a", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm1-b", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm1-c", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Pending"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "hotplug disk pod with virt-launcher (should succeed)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-abc", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createHotplugDiskPod("hp-volume-xyz", "default", "test-pvc", "virt-launcher-vm1-abc", "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "hotplug disks from different VMs (should fail)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-abc", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createHotplugDiskPod("hp-volume-vm1", "default", "test-pvc", "virt-launcher-vm1-abc", "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm2-xyz", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Running"),
|
||||
+ createHotplugDiskPod("hp-volume-vm2", "default", "test-pvc", "virt-launcher-vm2-xyz", "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: true,
|
||||
+ errorMsg: "different VMs",
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ for _, tc := range testCases {
|
||||
+ t.Run(tc.name, func(t *testing.T) {
|
||||
+ // Create fake dynamic client with test pods and PV
|
||||
+ scheme := runtime.NewScheme()
|
||||
+
|
||||
+ // Create PV object that references the PVC
|
||||
+ pv := createUnstructuredPV("test-volume-id", tc.namespace, tc.pvcName)
|
||||
+
|
||||
+ objects := make([]runtime.Object, 0, len(tc.pods)+1)
|
||||
+ objects = append(objects, pv)
|
||||
+
|
||||
+ for _, pod := range tc.pods {
|
||||
+ objects = append(objects, pod)
|
||||
+ }
|
||||
+
|
||||
+ gvrToListKind := map[schema.GroupVersionResource]string{
|
||||
+ PodGVR: "PodList",
|
||||
+ PVGVR: "PersistentVolumeList",
|
||||
+ }
|
||||
+ client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, objects...)
|
||||
+
|
||||
+ // Create logger
|
||||
+ logger := logrus.NewEntry(logrus.New())
|
||||
+ logger.Logger.SetLevel(logrus.DebugLevel)
|
||||
+
|
||||
+ // Run validation
|
||||
+ vmName, err := ValidateRWXBlockAttachment(context.Background(), client, logger, "test-volume-id")
|
||||
+
|
||||
+ if tc.expectError {
|
||||
+ assert.Error(t, err)
|
||||
+
|
||||
+ if tc.errorMsg != "" {
|
||||
+ assert.Contains(t, err.Error(), tc.errorMsg)
|
||||
+ }
|
||||
+ } else {
|
||||
+ assert.NoError(t, err)
|
||||
+ // VM name is returned when there are pods using the volume
|
||||
+ if len(tc.pods) > 0 {
|
||||
+ assert.NotEmpty(t, vmName)
|
||||
+ }
|
||||
+ }
|
||||
+ })
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func TestValidateRWXBlockAttachmentNoKubeClient(t *testing.T) {
|
||||
+ // When not running in Kubernetes (no client), validation should be skipped
|
||||
+ logger := logrus.NewEntry(logrus.New())
|
||||
+
|
||||
+ vmName, err := ValidateRWXBlockAttachment(context.Background(), nil, logger, "test-volume-id")
|
||||
+ assert.NoError(t, err)
|
||||
+ assert.Empty(t, vmName)
|
||||
+}
|
||||
+
|
||||
+func TestValidateRWXBlockAttachmentPVNotFound(t *testing.T) {
|
||||
+ // When PV is not found, validation should be skipped with warning
|
||||
+ scheme := runtime.NewScheme()
|
||||
+
|
||||
+ gvrToListKind := map[schema.GroupVersionResource]string{
|
||||
+ PodGVR: "PodList",
|
||||
+ PVGVR: "PersistentVolumeList",
|
||||
+ }
|
||||
+ client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind)
|
||||
+
|
||||
+ logger := logrus.NewEntry(logrus.New())
|
||||
+ logger.Logger.SetLevel(logrus.DebugLevel)
|
||||
+
|
||||
+ vmName, err := ValidateRWXBlockAttachment(context.Background(), client, logger, "non-existent-pv")
|
||||
+ assert.NoError(t, err)
|
||||
+ assert.Empty(t, vmName)
|
||||
+}
|
||||
+
|
||||
+// createUnstructuredPod creates an unstructured pod object for testing.
|
||||
+func createUnstructuredPod(name, namespace, pvcName string, labels map[string]string, phase string) *unstructured.Unstructured {
|
||||
+ pod := &unstructured.Unstructured{
|
||||
+ Object: map[string]interface{}{
|
||||
+ "apiVersion": "v1",
|
||||
+ "kind": "Pod",
|
||||
+ "metadata": map[string]interface{}{
|
||||
+ "name": name,
|
||||
+ "namespace": namespace,
|
||||
+ "labels": toStringInterfaceMap(labels),
|
||||
+ },
|
||||
+ "spec": map[string]interface{}{
|
||||
+ "volumes": []interface{}{
|
||||
+ map[string]interface{}{
|
||||
+ "name": "data",
|
||||
+ "persistentVolumeClaim": map[string]interface{}{
|
||||
+ "claimName": pvcName,
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ "status": map[string]interface{}{
|
||||
+ "phase": phase,
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ return pod
|
||||
+}
|
||||
+
|
||||
+// createUnstructuredPV creates an unstructured PersistentVolume object for testing.
|
||||
+func createUnstructuredPV(name, pvcNamespace, pvcName string) *unstructured.Unstructured {
|
||||
+ pv := &unstructured.Unstructured{
|
||||
+ Object: map[string]interface{}{
|
||||
+ "apiVersion": "v1",
|
||||
+ "kind": "PersistentVolume",
|
||||
+ "metadata": map[string]interface{}{
|
||||
+ "name": name,
|
||||
+ },
|
||||
+ "spec": map[string]interface{}{
|
||||
+ "claimRef": map[string]interface{}{
|
||||
+ "name": pvcName,
|
||||
+ "namespace": pvcNamespace,
|
||||
+ },
|
||||
+ "csi": map[string]interface{}{
|
||||
+ "volumeHandle": name,
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ return pv
|
||||
+}
|
||||
+
|
||||
+// toStringInterfaceMap converts map[string]string to map[string]interface{}.
|
||||
+func toStringInterfaceMap(m map[string]string) map[string]interface{} {
|
||||
+ result := make(map[string]interface{})
|
||||
+
|
||||
+ for k, v := range m {
|
||||
+ result[k] = v
|
||||
+ }
|
||||
+
|
||||
+ return result
|
||||
+}
|
||||
+
|
||||
+// createHotplugDiskPod creates a hotplug disk pod that references a virt-launcher pod via ownerReferences.
|
||||
+func createHotplugDiskPod(name, namespace, pvcName, ownerPodName, phase string) *unstructured.Unstructured {
|
||||
+ pod := &unstructured.Unstructured{
|
||||
+ Object: map[string]interface{}{
|
||||
+ "apiVersion": "v1",
|
||||
+ "kind": "Pod",
|
||||
+ "metadata": map[string]interface{}{
|
||||
+ "name": name,
|
||||
+ "namespace": namespace,
|
||||
+ "labels": map[string]interface{}{
|
||||
+ "kubevirt.io": "hotplug-disk",
|
||||
+ },
|
||||
+ "ownerReferences": []interface{}{
|
||||
+ map[string]interface{}{
|
||||
+ "apiVersion": "v1",
|
||||
+ "kind": "Pod",
|
||||
+ "name": ownerPodName,
|
||||
+ "controller": true,
|
||||
+ "blockOwnerDeletion": true,
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ "spec": map[string]interface{}{
|
||||
+ "volumes": []interface{}{
|
||||
+ map[string]interface{}{
|
||||
+ "name": "data",
|
||||
+ "persistentVolumeClaim": map[string]interface{}{
|
||||
+ "claimName": pvcName,
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ "status": map[string]interface{}{
|
||||
+ "phase": phase,
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ return pod
|
||||
+}
|
||||
@@ -21,10 +21,6 @@ spec:
|
||||
|
||||
scrapeInterval: 30s
|
||||
selectAllByDefault: false
|
||||
{{- with .Values.vmagent.inlineScrapeConfig }}
|
||||
inlineScrapeConfig: |
|
||||
{{- . | nindent 4 }}
|
||||
{{- end }}
|
||||
podScrapeNamespaceSelector:
|
||||
matchLabels:
|
||||
namespace.cozystack.io/monitoring: {{ .Release.Namespace }}
|
||||
|
||||
@@ -79,8 +79,4 @@ vmagent:
|
||||
remoteWrite:
|
||||
urls:
|
||||
- http://vminsert-shortterm:8480/insert/0/prometheus
|
||||
- http://vminsert-longterm:8480/insert/0/prometheus
|
||||
## inlineScrapeConfig: |
|
||||
## - job_name: "custom"
|
||||
## static_configs:
|
||||
## - targets: ["my-service:9090"]
|
||||
- http://vminsert-longterm:8480/insert/0/prometheus
|
||||
@@ -8,7 +8,7 @@ spec:
|
||||
singular: tenant
|
||||
plural: tenants
|
||||
openAPISchema: |-
|
||||
{"title":"Chart Values","type":"object","properties":{"etcd":{"description":"Deploy own Etcd cluster.","type":"boolean","default":false},"host":{"description":"The hostname used to access tenant services (defaults to using the tenant name as a subdomain for its parent tenant host).","type":"string","default":""},"ingress":{"description":"Deploy own Ingress Controller.","type":"boolean","default":false},"monitoring":{"description":"Deploy own Monitoring Stack.","type":"boolean","default":false},"resourceQuotas":{"description":"Define resource quotas for the tenant.","type":"object","default":{},"additionalProperties":{"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"seaweedfs":{"description":"Deploy own SeaweedFS.","type":"boolean","default":false}}}
|
||||
{"title":"Chart Values","type":"object","properties":{"etcd":{"description":"Deploy own Etcd cluster.","type":"boolean","default":false},"host":{"description":"The hostname used to access tenant services (defaults to using the tenant name as a subdomain for its parent tenant host).","type":"string","default":""},"ingress":{"description":"Deploy own Ingress Controller.","type":"boolean","default":false},"monitoring":{"description":"Deploy own Monitoring Stack.","type":"boolean","default":false},"resourceQuotas":{"description":"Define resource quotas for the tenant.","type":"object","default":{},"additionalProperties":{"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"schedulingClass":{"description":"The name of a SchedulingClass CR to apply scheduling constraints for this tenant's workloads.","type":"string","default":""},"seaweedfs":{"description":"Deploy own SeaweedFS.","type":"boolean","default":false}}}
|
||||
release:
|
||||
prefix: tenant-
|
||||
labels:
|
||||
@@ -23,7 +23,7 @@ spec:
|
||||
plural: Tenants
|
||||
description: Separated tenant namespace
|
||||
icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzQwMykiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zNDAzKSI+CjxwYXRoIGQ9Ik03MiAyOUM2Ni4zOTI2IDI5IDYxLjAxNDggMzEuMjM4OCA1Ny4wNDk3IDM1LjIyNEM1My4wODQ3IDM5LjIwOTEgNTAuODU3MSA0NC42MTQxIDUwLjg1NzEgNTAuMjVDNTAuODU3MSA1NS44ODU5IDUzLjA4NDcgNjEuMjkwOSA1Ny4wNDk3IDY1LjI3NkM2MS4wMTQ4IDY5LjI2MTIgNjYuMzkyNiA3MS41IDcyIDcxLjVDNzcuNjA3NCA3MS41IDgyLjk4NTIgNjkuMjYxMiA4Ni45NTAzIDY1LjI3NkM5MC45MTUzIDYxLjI5MDkgOTMuMTQyOSA1NS44ODU5IDkzLjE0MjkgNTAuMjVDOTMuMTQyOSA0NC42MTQxIDkwLjkxNTMgMzkuMjA5MSA4Ni45NTAzIDM1LjIyNEM4Mi45ODUyIDMxLjIzODggNzcuNjA3NCAyOSA3MiAyOVpNNjAuOTgyNiA4My4zMDM3QzYwLjQ1NCA4Mi41ODk4IDU5LjU5NTEgODIuMTkxNCA1OC43MTk2IDgyLjI3NDRDNDUuMzg5NyA4My43MzU0IDM1IDk1LjEwNzQgMzUgMTA4LjkwM0MzNSAxMTEuNzI2IDM3LjI3OTUgMTE0IDQwLjA3MSAxMTRIMTAzLjkyOUMxMDYuNzM3IDExNCAxMDkgMTExLjcwOSAxMDkgMTA4LjkwM0MxMDkgOTUuMTA3NCA5OC42MTAzIDgzLjc1MiA4NS4yNjM4IDgyLjI5MUM4NC4zODg0IDgyLjE5MTQgODMuNTI5NSA4Mi42MDY0IDgzLjAwMDkgODMuMzIwM0w3NC4wOTc4IDk1LjI0MDJDNzMuMDQwNiA5Ni42NTE0IDcwLjkyNjMgOTYuNjUxNCA2OS44NjkyIDk1LjI0MDJMNjAuOTY2MSA4My4zMjAzTDYwLjk4MjYgODMuMzAzN1oiIGZpbGw9ImJsYWNrIi8+CjwvZz4KPGRlZnM+CjxsaW5lYXJHcmFkaWVudCBpZD0icGFpbnQwX2xpbmVhcl82ODdfMzQwMyIgeDE9IjcyIiB5MT0iMTQ0IiB4Mj0iLTEuMjgxN2UtMDUiIHkyPSI0IiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+CjxzdG9wIHN0b3AtY29sb3I9IiNDMEQ2RkYiLz4KPHN0b3Agb2Zmc2V0PSIwLjMiIHN0b3AtY29sb3I9IiNDNERBRkYiLz4KPHN0b3Agb2Zmc2V0PSIwLjY1IiBzdG9wLWNvbG9yPSIjRDNFOUZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iI0U5RkZGRiIvPgo8L2xpbmVhckdyYWRpZW50Pgo8Y2xpcFBhdGggaWQ9ImNsaXAwXzY4N18zNDAzIj4KPHJlY3Qgd2lkdGg9Ijc0IiBoZWlnaHQ9Ijg1IiBmaWxsPSJ3aGl0ZSIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoMzUgMjkpIi8+CjwvY2xpcFBhdGg+CjwvZGVmcz4KPC9zdmc+Cg==
|
||||
keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "etcd"], ["spec", "monitoring"], ["spec", "ingress"], ["spec", "seaweedfs"], ["spec", "resourceQuotas"]]
|
||||
keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "etcd"], ["spec", "monitoring"], ["spec", "ingress"], ["spec", "seaweedfs"], ["spec", "schedulingClass"], ["spec", "resourceQuotas"]]
|
||||
secrets:
|
||||
exclude: []
|
||||
include: []
|
||||
|
||||
Reference in New Issue
Block a user