mirror of
https://github.com/outbackdingo/cozystack.git
synced 2026-01-27 10:18:39 +00:00
Merge branch 'main' into vnc
Signed-off-by: Timofei Larkin <lllamnyp@gmail.com>
This commit is contained in:
@@ -80,58 +80,41 @@ EOF
|
||||
# Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute)
|
||||
kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
|
||||
# Get the admin kubeconfig and save it to a file
|
||||
kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig
|
||||
kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig-${test_name}
|
||||
|
||||
# Update the kubeconfig to use localhost for the API server
|
||||
yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig
|
||||
yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig-${test_name}
|
||||
|
||||
|
||||
# Set up port forwarding to the Kubernetes API server for a 200 second timeout
|
||||
bash -c 'timeout 300s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &'
|
||||
# Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
|
||||
timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done'
|
||||
timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done'
|
||||
|
||||
# Wait for the nodes to be ready (timeout after 2 minutes)
|
||||
timeout 3m bash -c '
|
||||
until [ "$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath="{.items[*].metadata.name}" | wc -w)" -eq 2 ]; do
|
||||
until [ "$(kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' get nodes -o jsonpath="{.items[*].metadata.name}" | wc -w)" -eq 2 ]; do
|
||||
sleep 2
|
||||
done
|
||||
'
|
||||
# Verify the nodes are ready
|
||||
kubectl --kubeconfig tenantkubeconfig wait node --all --timeout=2m --for=condition=Ready
|
||||
kubectl --kubeconfig tenantkubeconfig get nodes -o wide
|
||||
kubectl --kubeconfig tenantkubeconfig-${test_name} wait node --all --timeout=2m --for=condition=Ready
|
||||
kubectl --kubeconfig tenantkubeconfig-${test_name} get nodes -o wide
|
||||
|
||||
# Verify the kubelet version matches what we expect
|
||||
versions=$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}')
|
||||
versions=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" \
|
||||
get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}')
|
||||
|
||||
node_ok=true
|
||||
|
||||
case "$k8s_version" in
|
||||
v1.32*)
|
||||
echo "⚠️ TODO: Temporary stub — allowing nodes with v1.33 while k8s_version is v1.32"
|
||||
;;
|
||||
esac
|
||||
|
||||
|
||||
for v in $versions; do
|
||||
case "$k8s_version" in
|
||||
v1.32|v1.32.*)
|
||||
case "$v" in
|
||||
v1.32 | v1.32.* | v1.32-* | v1.33 | v1.33.* | v1.33-*)
|
||||
;;
|
||||
*)
|
||||
node_ok=false
|
||||
break
|
||||
;;
|
||||
esac
|
||||
case "$v" in
|
||||
"${k8s_version}" | "${k8s_version}".* | "${k8s_version}"-*)
|
||||
# acceptable
|
||||
;;
|
||||
*)
|
||||
case "$v" in
|
||||
"${k8s_version}" | "${k8s_version}".* | "${k8s_version}"-*)
|
||||
;;
|
||||
*)
|
||||
node_ok=false
|
||||
break
|
||||
;;
|
||||
esac
|
||||
node_ok=false
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
@@ -19,3 +19,26 @@
|
||||
curl -sS --fail 'http://localhost:21234/openapi/v2?timeout=32s' -H 'Accept: application/com.github.proto-openapi.spec.v2@v1.0+protobuf' > /dev/null
|
||||
)
|
||||
}
|
||||
|
||||
@test "Test kinds" {
|
||||
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/tenants | jq -r '.kind')
|
||||
if [ "$val" != "TenantList" ]; then
|
||||
echo "Expected kind to be TenantList, got $val"
|
||||
exit 1
|
||||
fi
|
||||
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/tenants | jq -r '.items[0].kind')
|
||||
if [ "$val" != "Tenant" ]; then
|
||||
echo "Expected kind to be Tenant, got $val"
|
||||
exit 1
|
||||
fi
|
||||
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/ingresses | jq -r '.kind')
|
||||
if [ "$val" != "IngressList" ]; then
|
||||
echo "Expected kind to be IngressList, got $val"
|
||||
exit 1
|
||||
fi
|
||||
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/ingresses | jq -r '.items[0].kind')
|
||||
if [ "$val" != "Ingress" ]; then
|
||||
echo "Expected kind to be Ingress, got $val"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -153,6 +153,27 @@ func detailsTab(kind, endpoint, schemaJSON string, keysOrder [][]string) map[str
|
||||
}),
|
||||
paramsList,
|
||||
}
|
||||
if kind == "VirtualPrivateCloud" {
|
||||
rightColStack = append(rightColStack,
|
||||
antdFlexVertical("vpc-subnets-block", 4, []any{
|
||||
antdText("vpc-subnets-label", true, "Subnets", nil),
|
||||
map[string]any{
|
||||
"type": "EnrichedTable",
|
||||
"data": map[string]any{
|
||||
"id": "vpc-subnets-table",
|
||||
"baseprefix": "/openapi-ui",
|
||||
"clusterNamePartOfUrl": "{2}",
|
||||
"customizationId": "virtualprivatecloud-subnets",
|
||||
"fetchUrl": "/api/clusters/{2}/k8s/api/v1/namespaces/{3}/configmaps",
|
||||
"fieldSelector": map[string]any{
|
||||
"metadata.name": "virtualprivatecloud-{6}-subnets",
|
||||
},
|
||||
"pathToItems": []any{"items"},
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"key": "details",
|
||||
|
||||
@@ -182,6 +182,13 @@ func CreateAllCustomColumnsOverrides() []*dashboardv1alpha1.CustomColumnsOverrid
|
||||
createTimestampColumn("Created", ".metadata.creationTimestamp"),
|
||||
}),
|
||||
|
||||
// Virtual private cloud subnets
|
||||
createCustomColumnsOverride("virtualprivatecloud-subnets", []any{
|
||||
createFlatMapColumn("Data", ".data"),
|
||||
createStringColumn("Subnet Parameters", "_flatMapData_Key"),
|
||||
createStringColumn("Values", "_flatMapData_Value"),
|
||||
}),
|
||||
|
||||
// Factory ingress details rules
|
||||
createCustomColumnsOverride("factory-kube-ingress-details-rules", []any{
|
||||
createStringColumn("Host", ".host"),
|
||||
|
||||
@@ -1 +1 @@
|
||||
ghcr.io/cozystack/cozystack/nginx-cache:0.0.0@sha256:50ac1581e3100bd6c477a71161cb455a341ffaf9e5e2f6086802e4e25271e8af
|
||||
ghcr.io/cozystack/cozystack/nginx-cache:0.0.0@sha256:b7633717cd7449c0042ae92d8ca9b36e4d69566561f5c7d44e21058e7d05c6d5
|
||||
|
||||
@@ -1 +1 @@
|
||||
ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:c8b08084a86251cdd18e237de89b695bca0e4f7eb1f1f6ddc2b903b4d74ea5ff
|
||||
ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:f94f034f29aecf81d34ce87efb5ef0e82afc218b809494c4b477343965594289
|
||||
|
||||
@@ -182,6 +182,33 @@ metadata:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
files:
|
||||
- path: /usr/bin/update-k8s.sh
|
||||
owner: root:root
|
||||
permissions: "0755"
|
||||
content: |
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Expected to be passed in via preKubeadmCommands
|
||||
: "${KUBELET_VERSION:?KUBELET_VERSION must be set, e.g. v1.31.0}"
|
||||
|
||||
ARCH="$(uname -m)"
|
||||
case "${ARCH}" in
|
||||
x86_64) ARCH=amd64 ;;
|
||||
aarch64) ARCH=arm64 ;;
|
||||
esac
|
||||
|
||||
# Use your internal mirror here for real-world use.
|
||||
BASE_URL="https://dl.k8s.io/release/${KUBELET_VERSION}/bin/linux/${ARCH}"
|
||||
|
||||
echo "Installing kubelet and kubeadm ${KUBELET_VERSION} for ${ARCH}..."
|
||||
curl -fsSL "${BASE_URL}/kubelet" -o /root/kubelet
|
||||
curl -fsSL "${BASE_URL}/kubeadm" -o /root/kubeadm
|
||||
chmod 0755 /root/kubelet
|
||||
chmod 0755 /root/kubeadm
|
||||
if /root/kubelet --version ; then mv /root/kubelet /usr/bin/kubelet ; fi
|
||||
if /root/kubeadm version ; then mv /root/kubeadm /usr/bin/kubeadm ; fi
|
||||
diskSetup:
|
||||
filesystems:
|
||||
- device: /dev/vdb
|
||||
@@ -205,6 +232,7 @@ spec:
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
preKubeadmCommands:
|
||||
- KUBELET_VERSION={{ include "kubernetes.versionMap" $}} /usr/bin/update-k8s.sh || true
|
||||
- sed -i 's|root:x:|root::|' /etc/passwd
|
||||
- systemctl stop containerd.service
|
||||
- mkdir -p /ephemeral/kubelet /ephemeral/containerd
|
||||
|
||||
@@ -6,11 +6,11 @@ metadata:
|
||||
"helm.sh/hook": post-delete
|
||||
"helm.sh/hook-weight": "10"
|
||||
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed
|
||||
name: {{ .Release.Name }}-datavolume-cleanup
|
||||
name: {{ .Release.Name }}-cleanup
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: {{ .Release.Name }}-datavolume-cleanup
|
||||
serviceAccountName: {{ .Release.Name }}-cleanup
|
||||
restartPolicy: Never
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
@@ -28,12 +28,17 @@ spec:
|
||||
-l "cluster.x-k8s.io/cluster-name={{ .Release.Name }}"
|
||||
--ignore-not-found=true
|
||||
|
||||
kubectl -n {{ .Release.Namespace }} delete services
|
||||
-l "cluster.x-k8s.io/cluster-name={{ .Release.Name }}"
|
||||
--field-selector spec.type=LoadBalancer
|
||||
--ignore-not-found=true
|
||||
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-datavolume-cleanup
|
||||
name: {{ .Release.Name }}-cleanup
|
||||
annotations:
|
||||
helm.sh/hook: post-delete
|
||||
helm.sh/hook-delete-policy: before-hook-creation,hook-failed,hook-succeeded
|
||||
@@ -46,7 +51,7 @@ metadata:
|
||||
"helm.sh/hook": post-delete
|
||||
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed
|
||||
"helm.sh/hook-weight": "5"
|
||||
name: {{ .Release.Name }}-datavolume-cleanup
|
||||
name: {{ .Release.Name }}-cleanup
|
||||
rules:
|
||||
- apiGroups:
|
||||
- "cdi.kubevirt.io"
|
||||
@@ -56,6 +61,14 @@ rules:
|
||||
- get
|
||||
- list
|
||||
- delete
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- services
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- delete
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
@@ -64,13 +77,13 @@ metadata:
|
||||
"helm.sh/hook": post-delete
|
||||
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed
|
||||
"helm.sh/hook-weight": "5"
|
||||
name: {{ .Release.Name }}-datavolume-cleanup
|
||||
name: {{ .Release.Name }}-cleanup
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ .Release.Name }}-datavolume-cleanup
|
||||
name: {{ .Release.Name }}-cleanup
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ .Release.Name }}-datavolume-cleanup
|
||||
name: {{ .Release.Name }}-cleanup
|
||||
namespace: {{ .Release.Namespace }}
|
||||
|
||||
@@ -122,7 +122,7 @@ metadata:
|
||||
name: {{ include "tenant.name" . }}-view
|
||||
namespace: {{ include "tenant.name" . }}
|
||||
subjects:
|
||||
{{ include "cozy-lib.rbac.subjectsForTenant" (list "view" (include "tenant.name" .)) | nindent 2 }}
|
||||
{{ include "cozy-lib.rbac.subjectsForTenantAndAccessLevel" (list "view" (include "tenant.name" .)) | nindent 2 }}
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: {{ include "tenant.name" . }}-view
|
||||
@@ -200,7 +200,7 @@ metadata:
|
||||
name: {{ include "tenant.name" . }}-use
|
||||
namespace: {{ include "tenant.name" . }}
|
||||
subjects:
|
||||
{{ include "cozy-lib.rbac.subjectsForTenant" (list "use" (include "tenant.name" .)) | nindent 2 }}
|
||||
{{ include "cozy-lib.rbac.subjectsForTenantAndAccessLevel" (list "use" (include "tenant.name" .)) | nindent 2 }}
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: {{ include "tenant.name" . }}-use
|
||||
@@ -299,7 +299,7 @@ metadata:
|
||||
name: {{ include "tenant.name" . }}-admin
|
||||
namespace: {{ include "tenant.name" . }}
|
||||
subjects:
|
||||
{{ include "cozy-lib.rbac.subjectsForTenant" (list "admin" (include "tenant.name" .)) | nindent 2 }}
|
||||
{{ include "cozy-lib.rbac.subjectsForTenantAndAccessLevel" (list "admin" (include "tenant.name" .)) | nindent 2 }}
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: {{ include "tenant.name" . }}-admin
|
||||
@@ -373,7 +373,7 @@ metadata:
|
||||
name: {{ include "tenant.name" . }}-super-admin
|
||||
namespace: {{ include "tenant.name" . }}
|
||||
subjects:
|
||||
{{ include "cozy-lib.rbac.subjectsForTenant" (list "super-admin" (include "tenant.name" .) ) | nindent 2 }}
|
||||
{{ include "cozy-lib.rbac.subjectsForTenantAndAccessLevel" (list "super-admin" (include "tenant.name" .) ) | nindent 2 }}
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: {{ include "tenant.name" . }}-super-admin
|
||||
|
||||
@@ -19,7 +19,7 @@ Subnet name and ip address range must be unique within a VPC.
|
||||
Subnet ip address space must not overlap with the default management network ip address range, subsets of 172.16.0.0/12 are recommended.
|
||||
Currently there are no fail-safe checks, however they are planned for the future.
|
||||
|
||||
Different VPCs may have subnets with ovelapping ip address ranges.
|
||||
Different VPCs may have subnets with overlapping ip address ranges.
|
||||
|
||||
A VM or a pod may be connected to multiple secondary Subnets at once. Each secondary connection will be represented as an additional network interface.
|
||||
|
||||
|
||||
1
packages/apps/vpc/charts/cozy-lib
Symbolic link
1
packages/apps/vpc/charts/cozy-lib
Symbolic link
@@ -0,0 +1 @@
|
||||
../../../library/cozy-lib
|
||||
@@ -60,13 +60,33 @@ kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ $.Release.Name }}-subnets
|
||||
labels:
|
||||
apps.cozystack.io/application.group: apps.cozystack.io
|
||||
apps.cozystack.io/application.kind: VirtualPrivateCloud
|
||||
apps.cozystack.io/application.name: {{ trimPrefix "virtualprivatecloud-" .Release.Name }}
|
||||
cozystack.io/vpcId: {{ $vpcId }}
|
||||
cozystack.io/tenantName: {{ $.Release.Namespace }}
|
||||
data:
|
||||
{{- range $subnetName, $subnetConfig := .Values.subnets }}
|
||||
{{ $subnetName }}: |-
|
||||
subnetName: {{ $subnetName }}
|
||||
subnetId: {{ print "subnet-" (print $.Release.Namespace "/" $vpcId "/" $subnetName | sha256sum | trunc 8) }}
|
||||
subnetCIDR: {{ $subnetConfig.cidr }}
|
||||
{{ $subnetName }}.ID: {{ print "subnet-" (print $.Release.Namespace "/" $vpcId "/" $subnetName | sha256sum | trunc 8) }}
|
||||
{{ $subnetName }}.CIDR: {{ $subnetConfig.cidr }}
|
||||
{{- end }}
|
||||
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-subnets"
|
||||
subjects: {{- include "cozy-lib.rbac.subjectsForTenantAndAccessLevel" (list "view" .Release.Namespace ) | nindent 2 }}
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: "{{ .Release.Name }}-subnets"
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-subnets"
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get","list","watch"]
|
||||
resourceNames: ["{{ .Release.Name }}-subnets"]
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
cozystack:
|
||||
image: ghcr.io/cozystack/cozystack/installer:v0.37.0@sha256:256c5a0f0ae2fc3ad6865b9fda74c42945b38a5384240fa29554617185b60556
|
||||
image: ghcr.io/cozystack/cozystack/installer:v0.38.0-alpha.1@sha256:ba25a3e1c3008e0bcf5e50b6ac2cce371cec880a7f4fe4b8de1abf621bb3017f
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
e2e:
|
||||
image: ghcr.io/cozystack/cozystack/e2e-sandbox:v0.37.0@sha256:10afd0a6c39248ec41d0e59ff1bc6c29bd0075b7cc9a512b01cf603ef39c33ea
|
||||
image: ghcr.io/cozystack/cozystack/e2e-sandbox:v0.38.0-alpha.1@sha256:51a6674dfb938bcd9ff2574e2147722db98c9eda788ab7d017ee4ca03da4231b
|
||||
|
||||
@@ -1 +1 @@
|
||||
ghcr.io/cozystack/cozystack/matchbox:v0.37.0@sha256:5cca5f56b755285aefa11b1052fe55e1aa83b25bae34aef80cdb77ff63091044
|
||||
ghcr.io/cozystack/cozystack/matchbox:v0.38.0-alpha.1@sha256:5396b4a3cc652d35a9348ebf957c823c650c8a72ccea693fbbe72df675dc9317
|
||||
|
||||
@@ -1 +1 @@
|
||||
ghcr.io/cozystack/cozystack/objectstorage-sidecar:v0.37.0@sha256:f166f09cdc9cdbb758209883819ab8261a3793bc1d7a6b6685efd5a2b2930847
|
||||
ghcr.io/cozystack/cozystack/objectstorage-sidecar:v0.38.0-alpha.1@sha256:b8475ec6c95cb26be8e4da0312b2a9d5498400373ad40d4ca168fb4b02ae8f94
|
||||
|
||||
@@ -1 +1 @@
|
||||
ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:7348bec610f08bd902c88c9a9f28fdd644727e2728a1e4103f88f0c99febd5e7
|
||||
ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:efbfe6d7f27d1e9d2334006343bb062825cc8fee0187145144746b8738e4fe4b
|
||||
|
||||
1
packages/system/cozystack-api/.gitignore
vendored
Normal file
1
packages/system/cozystack-api/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
apiserver.local.config/
|
||||
@@ -4,6 +4,18 @@ NAMESPACE=cozy-system
|
||||
include ../../../scripts/common-envs.mk
|
||||
include ../../../scripts/package.mk
|
||||
|
||||
run-local:
|
||||
openssl req -nodes -new -x509 -keyout /tmp/ca.key -out /tmp/ca.crt -subj "/CN=kube-ca"
|
||||
openssl req -out /tmp/client.csr -new -newkey rsa:2048 -nodes -keyout /tmp/client.key -subj "/C=US/ST=SomeState/L=L/OU=Dev/CN=development/O=system:masters"
|
||||
openssl x509 -req -days 365 -in /tmp/client.csr -CA /tmp/ca.crt -CAkey /tmp/ca.key -set_serial 01 -sha256 -out /tmp/client.crt
|
||||
openssl req -out /tmp/apiserver.csr -new -newkey rsa:2048 -nodes -keyout /tmp/apiserver.key -subj "/CN=cozystack-api" -config cozystack-api-openssl.cnf
|
||||
openssl x509 -req -days 365 -in /tmp/apiserver.csr -CA /tmp/ca.crt -CAkey /tmp/ca.key -set_serial 01 -sha256 -out /tmp/apiserver.crt -extensions v3_req -extfile cozystack-api-openssl.cnf
|
||||
CGO_ENABLED=0 go build -o /tmp/cozystack-api ../../../cmd/cozystack-api/main.go
|
||||
/tmp/cozystack-api --client-ca-file /tmp/ca.crt --tls-cert-file /tmp/apiserver.crt --tls-private-key-file /tmp/apiserver.key --secure-port 6443 --kubeconfig $(KUBECONFIG) --authorization-kubeconfig $(KUBECONFIG) --authentication-kubeconfig $(KUBECONFIG)
|
||||
|
||||
debug:
|
||||
dlv debug ../../../cmd/cozystack-api/main.go -- --client-ca-file /tmp/ca.crt --tls-cert-file /tmp/apiserver.crt --tls-private-key-file /tmp/apiserver.key --secure-port 6443 --kubeconfig $(KUBECONFIG) --authorization-kubeconfig $(KUBECONFIG) --authentication-kubeconfig $(KUBECONFIG)
|
||||
|
||||
image: image-cozystack-api
|
||||
|
||||
image-cozystack-api:
|
||||
|
||||
13
packages/system/cozystack-api/cozystack-api-openssl.cnf
Normal file
13
packages/system/cozystack-api/cozystack-api-openssl.cnf
Normal file
@@ -0,0 +1,13 @@
|
||||
[ req ]
|
||||
distinguished_name = req_distinguished_name
|
||||
req_extensions = v3_req
|
||||
prompt = no
|
||||
|
||||
[ req_distinguished_name ]
|
||||
CN = cozystack-api
|
||||
|
||||
[ v3_req ]
|
||||
subjectAltName = @alt_names
|
||||
|
||||
[ alt_names ]
|
||||
IP.1 = 127.0.0.1
|
||||
@@ -1,5 +1,5 @@
|
||||
cozystackAPI:
|
||||
image: ghcr.io/cozystack/cozystack/cozystack-api:v0.37.0@sha256:19d89e8afb90ce38ab7e42ecedfc28402f7c0b56f30957db957c5415132ff6ca
|
||||
image: ghcr.io/cozystack/cozystack/cozystack-api:v0.38.0-alpha.1@sha256:e4c028f802d2facbc1b9be0afc9856f5927a715edeaf7c004dd80ac8f00b2304
|
||||
localK8sAPIEndpoint:
|
||||
enabled: true
|
||||
replicas: 2
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
cozystackController:
|
||||
image: ghcr.io/cozystack/cozystack/cozystack-controller:v0.37.0@sha256:845b8e68cbc277c2303080bcd55597e4334610d396dad258ad56fd906530acc3
|
||||
image: ghcr.io/cozystack/cozystack/cozystack-controller:v0.38.0-alpha.1@sha256:971ca76078d9b81e3b02fd270147754af3eb5505b08aad0ead8d41ae9a3d0fae
|
||||
debug: false
|
||||
disableTelemetry: false
|
||||
cozystackVersion: "v0.37.0"
|
||||
cozystackVersion: "v0.38.0-alpha.1"
|
||||
cozystackAPIKind: "DaemonSet"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{{- $brandingConfig:= lookup "v1" "ConfigMap" "cozy-system" "cozystack-branding" }}
|
||||
|
||||
{{- $tenantText := "latest" }}
|
||||
{{- $tenantText := "v0.38.0-alpha.1" }}
|
||||
{{- $footerText := "Cozystack" }}
|
||||
{{- $titleText := "Cozystack Dashboard" }}
|
||||
{{- $logoText := "" }}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
openapiUI:
|
||||
image: ghcr.io/cozystack/cozystack/openapi-ui:latest@sha256:931f95b93bf7188b9a52f9d4b2a71979f3570340c860223efbc781339d792d34
|
||||
image: ghcr.io/cozystack/cozystack/openapi-ui:v0.38.0-alpha.1@sha256:901e8e3b86b72ae597e8f13efccb5493b23cd050e4c584ea00ce41da39c912cb
|
||||
openapiUIK8sBff:
|
||||
image: ghcr.io/cozystack/cozystack/openapi-ui-k8s-bff:latest@sha256:8386f0747266726afb2b30db662092d66b0af0370e3becd8bee9684125fa9cc9
|
||||
image: ghcr.io/cozystack/cozystack/openapi-ui-k8s-bff:v0.38.0-alpha.1@sha256:c5a7d369b0199b1a02ff3a9c5233835eb274b4a6253677cd5c459d4b1e88db4d
|
||||
tokenProxy:
|
||||
image: ghcr.io/cozystack/cozystack/token-proxy:latest@sha256:fad27112617bb17816702571e1f39d0ac3fe5283468d25eb12f79906cdab566b
|
||||
image: ghcr.io/cozystack/cozystack/token-proxy:v0.38.0-alpha.1@sha256:fad27112617bb17816702571e1f39d0ac3fe5283468d25eb12f79906cdab566b
|
||||
|
||||
@@ -3,7 +3,7 @@ kamaji:
|
||||
deploy: false
|
||||
image:
|
||||
pullPolicy: IfNotPresent
|
||||
tag: v0.37.0@sha256:9f4fd5045ede2909fbaf2572e4138fcbd8921071ecf8f08446257fddd0e6f655
|
||||
tag: v0.38.0-alpha.1@sha256:912674a7ca09b745cd7921661bfe3ae8c66b6d0d5a0d842a3219e0997349a905
|
||||
repository: ghcr.io/cozystack/cozystack/kamaji
|
||||
resources:
|
||||
limits:
|
||||
@@ -13,4 +13,4 @@ kamaji:
|
||||
cpu: 100m
|
||||
memory: 100Mi
|
||||
extraArgs:
|
||||
- --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v0.37.0@sha256:9f4fd5045ede2909fbaf2572e4138fcbd8921071ecf8f08446257fddd0e6f655
|
||||
- --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v0.38.0-alpha.1@sha256:912674a7ca09b745cd7921661bfe3ae8c66b6d0d5a0d842a3219e0997349a905
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
portSecurity: true
|
||||
routes: ""
|
||||
image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v0.37.0@sha256:9950614571ea77a55925eba0839b6b12c8e5a7a30b8858031a8c6050f261af1a
|
||||
image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v0.38.0-alpha.1@sha256:a62d3bca304a51f0945f88d855a6afbec3f4fef4b8b35d55d6c16d162efeb758
|
||||
ovnCentralName: ovn-central
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
portSecurity: true
|
||||
routes: ""
|
||||
image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v0.37.0@sha256:7e63205708e607ce2cedfe2a2cafd323ca51e3ebc71244a21ff6f9016c6c87bc
|
||||
image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v0.38.0-alpha.1@sha256:095e979bcbaa275fd5ced339cd8f415b87dd151135735e37568c751da120137e
|
||||
|
||||
@@ -44,7 +44,7 @@ kube-ovn:
|
||||
memory: "50Mi"
|
||||
limits:
|
||||
cpu: "1000m"
|
||||
memory: "1Gi"
|
||||
memory: "2Gi"
|
||||
kube-ovn-pinger:
|
||||
requests:
|
||||
cpu: "10m"
|
||||
@@ -65,4 +65,4 @@ global:
|
||||
images:
|
||||
kubeovn:
|
||||
repository: kubeovn
|
||||
tag: v1.14.5@sha256:af10da442a0c6dc7df47a0ef752e2eb5c247bb0b43069fdfcb2aa51511185ea2
|
||||
tag: v1.14.11@sha256:6b26ba0947732e8b33ff921057ae768bea36c230843ea34ce93b317157597831
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
storageClass: replicated
|
||||
csiDriver:
|
||||
image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:c8b08084a86251cdd18e237de89b695bca0e4f7eb1f1f6ddc2b903b4d74ea5ff
|
||||
image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:f94f034f29aecf81d34ce87efb5ef0e82afc218b809494c4b477343965594289
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
lineageControllerWebhook:
|
||||
image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v0.37.0@sha256:845b8e68cbc277c2303080bcd55597e4334610d396dad258ad56fd906530acc3
|
||||
image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v0.38.0-alpha.1@sha256:590daccdafda652eea697c691f5695e9f322056971e5462351911bb8931d7573
|
||||
debug: false
|
||||
localK8sAPIEndpoint:
|
||||
enabled: true
|
||||
|
||||
@@ -154,7 +154,7 @@ spec:
|
||||
serviceAccountName: multus
|
||||
containers:
|
||||
- name: kube-multus
|
||||
image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.2-thick
|
||||
image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.3-thick
|
||||
command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
|
||||
resources:
|
||||
requests:
|
||||
@@ -162,7 +162,7 @@ spec:
|
||||
memory: "100Mi"
|
||||
limits:
|
||||
cpu: "100m"
|
||||
memory: "100Mi"
|
||||
memory: "300Mi"
|
||||
securityContext:
|
||||
privileged: true
|
||||
terminationMessagePolicy: FallbackToLogsOnError
|
||||
@@ -201,11 +201,13 @@ spec:
|
||||
fieldPath: spec.nodeName
|
||||
initContainers:
|
||||
- name: install-multus-binary
|
||||
image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.2-thick
|
||||
image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.3-thick
|
||||
command:
|
||||
- "sh"
|
||||
- "-c"
|
||||
- "cp /usr/src/multus-cni/bin/multus-shim /host/opt/cni/bin/multus-shim && cp /usr/src/multus-cni/bin/passthru /host/opt/cni/bin/passthru"
|
||||
- "/usr/src/multus-cni/bin/install_multus"
|
||||
- "-d"
|
||||
- "/host/opt/cni/bin"
|
||||
- "-t"
|
||||
- "thick"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "10m"
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
objectstorage:
|
||||
controller:
|
||||
image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v0.37.0@sha256:5f2eed05d19ba971806374834cb16ca49282aac76130194c00b213c79ce3e10d"
|
||||
image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v0.38.0-alpha.1@sha256:eb55bf7d90d1be91dc037bfa3781a9bebf2462d4d10bce3e45b941fe3258e8bf"
|
||||
|
||||
@@ -124,7 +124,7 @@ seaweedfs:
|
||||
bucketClassName: "seaweedfs"
|
||||
region: ""
|
||||
sidecar:
|
||||
image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:latest@sha256:f4ec2b5ec8183870e710b32fa11b3af5d97fa664572df5edcff4b593b00f9a7b"
|
||||
image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v0.38.0-alpha.1@sha256:b8475ec6c95cb26be8e4da0312b2a9d5498400373ad40d4ca168fb4b02ae8f94"
|
||||
certificates:
|
||||
commonName: "SeaweedFS CA"
|
||||
ipAddresses: []
|
||||
|
||||
@@ -30,14 +30,16 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/apiserver/pkg/registry/rest"
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
"k8s.io/klog/v2"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/cache"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
|
||||
"github.com/cozystack/cozystack/pkg/apis/apps"
|
||||
appsinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
|
||||
coreinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
|
||||
"github.com/cozystack/cozystack/pkg/apis/core"
|
||||
coreinstall "github.com/cozystack/cozystack/pkg/apis/core/install"
|
||||
"github.com/cozystack/cozystack/pkg/config"
|
||||
cozyregistry "github.com/cozystack/cozystack/pkg/registry"
|
||||
applicationstorage "github.com/cozystack/cozystack/pkg/registry/apps/application"
|
||||
@@ -48,7 +50,8 @@ import (
|
||||
|
||||
var (
|
||||
// Scheme defines methods for serializing and deserializing API objects.
|
||||
Scheme = runtime.NewScheme()
|
||||
Scheme = runtime.NewScheme()
|
||||
mgrScheme = runtime.NewScheme()
|
||||
// Codecs provides methods for retrieving codecs and serializers for specific
|
||||
// versions and content types.
|
||||
Codecs = serializer.NewCodecFactory(Scheme)
|
||||
@@ -57,18 +60,23 @@ var (
|
||||
)
|
||||
|
||||
func init() {
|
||||
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{
|
||||
Development: true,
|
||||
// any other zap.Options tweaks
|
||||
})))
|
||||
klog.SetLogger(ctrl.Log.WithName("klog"))
|
||||
appsinstall.Install(Scheme)
|
||||
coreinstall.Install(Scheme)
|
||||
|
||||
// Register HelmRelease types.
|
||||
if err := helmv2.AddToScheme(Scheme); err != nil {
|
||||
if err := helmv2.AddToScheme(mgrScheme); err != nil {
|
||||
panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err))
|
||||
}
|
||||
|
||||
if err := corev1.AddToScheme(Scheme); err != nil {
|
||||
if err := corev1.AddToScheme(mgrScheme); err != nil {
|
||||
panic(fmt.Errorf("Failed to add core types to scheme: %w", err))
|
||||
}
|
||||
if err := rbacv1.AddToScheme(Scheme); err != nil {
|
||||
if err := rbacv1.AddToScheme(mgrScheme); err != nil {
|
||||
panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err))
|
||||
}
|
||||
// Add unversioned types.
|
||||
@@ -134,7 +142,7 @@ func (c completedConfig) New() (*CozyServer, error) {
|
||||
}
|
||||
|
||||
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
|
||||
Scheme: Scheme,
|
||||
Scheme: mgrScheme,
|
||||
Cache: cache.Options{SyncPeriod: &syncPeriod},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -164,7 +172,7 @@ func (c completedConfig) New() (*CozyServer, error) {
|
||||
}
|
||||
|
||||
cli := mgr.GetClient()
|
||||
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: Scheme})
|
||||
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: mgrScheme})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build watch client: %w", err)
|
||||
}
|
||||
|
||||
@@ -42,11 +42,11 @@ import (
|
||||
genericoptions "k8s.io/apiserver/pkg/server/options"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversionpkg "k8s.io/apiserver/pkg/util/version"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"k8s.io/component-base/featuregate"
|
||||
baseversion "k8s.io/component-base/version"
|
||||
netutils "k8s.io/utils/net"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
k8sconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
|
||||
)
|
||||
|
||||
// CozyServerOptions holds the state for the Cozy API server
|
||||
@@ -150,7 +150,7 @@ func (o *CozyServerOptions) Complete() error {
|
||||
return fmt.Errorf("failed to register types: %w", err)
|
||||
}
|
||||
|
||||
cfg, err := clientcmd.BuildConfigFromFlags("", "")
|
||||
cfg, err := k8sconfig.GetConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get kubeconfig: %w", err)
|
||||
}
|
||||
|
||||
@@ -142,9 +142,17 @@ func (r *REST) GetSingularName() string {
|
||||
// Create handles the creation of a new Application by converting it to a HelmRelease
|
||||
func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
|
||||
// Assert the object is of type Application
|
||||
app, ok := obj.(*appsv1alpha1.Application)
|
||||
us, ok := obj.(*unstructured.Unstructured)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("expected Application object, got %T", obj)
|
||||
return nil, fmt.Errorf("expected unstructured.Unstructured object, got %T", obj)
|
||||
}
|
||||
|
||||
app := &appsv1alpha1.Application{}
|
||||
|
||||
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(us.Object, app); err != nil {
|
||||
errMsg := fmt.Sprintf("returned unstructured.Unstructured object was not an Application")
|
||||
klog.Errorf(errMsg)
|
||||
return nil, fmt.Errorf(errMsg)
|
||||
}
|
||||
|
||||
// Convert Application to HelmRelease
|
||||
@@ -389,11 +397,9 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption
|
||||
}
|
||||
|
||||
// Explicitly set apiVersion and kind in unstructured object
|
||||
appList := &unstructured.UnstructuredList{}
|
||||
appList.SetAPIVersion("apps.cozystack.io/v1alpha1")
|
||||
appList.SetKind(r.kindName + "List")
|
||||
appList := r.NewList().(*unstructured.Unstructured)
|
||||
appList.SetResourceVersion(hrList.GetResourceVersion())
|
||||
appList.Items = items
|
||||
appList.Object["items"] = items
|
||||
|
||||
klog.V(6).Infof("Successfully listed %d Application resources in namespace %s", len(items), namespace)
|
||||
return appList, nil
|
||||
@@ -441,9 +447,16 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
|
||||
}
|
||||
|
||||
// Assert the new object is of type Application
|
||||
app, ok := newObj.(*appsv1alpha1.Application)
|
||||
us, ok := newObj.(*unstructured.Unstructured)
|
||||
if !ok {
|
||||
errMsg := fmt.Sprintf("expected Application object, got %T", newObj)
|
||||
errMsg := fmt.Sprintf("expected unstructured.Unstructured object, got %T", newObj)
|
||||
klog.Errorf(errMsg)
|
||||
return nil, false, fmt.Errorf(errMsg)
|
||||
}
|
||||
app := &appsv1alpha1.Application{}
|
||||
|
||||
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(us.Object, app); err != nil {
|
||||
errMsg := fmt.Sprintf("returned unstructured.Unstructured object was not an Application")
|
||||
klog.Errorf(errMsg)
|
||||
return nil, false, fmt.Errorf(errMsg)
|
||||
}
|
||||
@@ -516,14 +529,12 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
|
||||
klog.Errorf("Failed to convert Application to unstructured for resource %s: %v", convertedApp.GetName(), err)
|
||||
return nil, false, fmt.Errorf("failed to convert Application to unstructured: %v", err)
|
||||
}
|
||||
|
||||
// Explicitly set apiVersion and kind in unstructured object
|
||||
unstructuredApp["apiVersion"] = "apps.cozystack.io/v1alpha1"
|
||||
unstructuredApp["kind"] = r.kindName
|
||||
obj := &unstructured.Unstructured{Object: unstructuredApp}
|
||||
obj.SetGroupVersionKind(r.gvk)
|
||||
|
||||
klog.V(6).Infof("Returning patched Application object: %+v", unstructuredApp)
|
||||
|
||||
return &unstructured.Unstructured{Object: unstructuredApp}, false, nil
|
||||
return obj, false, nil
|
||||
}
|
||||
|
||||
// Delete removes an Application by deleting the corresponding HelmRelease
|
||||
@@ -723,11 +734,13 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio
|
||||
klog.Errorf("Failed to convert Application to unstructured: %v", err)
|
||||
continue
|
||||
}
|
||||
obj := &unstructured.Unstructured{Object: unstructuredApp}
|
||||
obj.SetGroupVersionKind(r.gvk)
|
||||
|
||||
// Create watch event with Application object
|
||||
appEvent := watch.Event{
|
||||
Type: event.Type,
|
||||
Object: &unstructured.Unstructured{Object: unstructuredApp},
|
||||
Object: obj,
|
||||
}
|
||||
|
||||
// Send event to custom watcher
|
||||
@@ -1060,6 +1073,34 @@ func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableO
|
||||
table = r.buildTableFromApplications(apps)
|
||||
table.ListMeta.ResourceVersion = obj.GetResourceVersion()
|
||||
case *unstructured.Unstructured:
|
||||
var apps []appsv1alpha1.Application
|
||||
for {
|
||||
var items interface{}
|
||||
var ok bool
|
||||
var objects []unstructured.Unstructured
|
||||
if items, ok = obj.Object["items"]; !ok {
|
||||
break
|
||||
}
|
||||
if objects, ok = items.([]unstructured.Unstructured); !ok {
|
||||
break
|
||||
}
|
||||
apps = make([]appsv1alpha1.Application, 0, len(objects))
|
||||
var a appsv1alpha1.Application
|
||||
for i := range objects {
|
||||
err := runtime.DefaultUnstructuredConverter.FromUnstructured(objects[i].Object, &a)
|
||||
if err != nil {
|
||||
klog.Errorf("Failed to convert Unstructured to Application: %v", err)
|
||||
continue
|
||||
}
|
||||
apps = append(apps, a)
|
||||
}
|
||||
break
|
||||
}
|
||||
if apps != nil {
|
||||
table = r.buildTableFromApplications(apps)
|
||||
table.ListMeta.ResourceVersion = obj.GetResourceVersion()
|
||||
break
|
||||
}
|
||||
var app appsv1alpha1.Application
|
||||
err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &app)
|
||||
if err != nil {
|
||||
@@ -1196,12 +1237,17 @@ func (r *REST) Destroy() {
|
||||
|
||||
// New creates a new instance of Application
|
||||
func (r *REST) New() runtime.Object {
|
||||
return &appsv1alpha1.Application{}
|
||||
obj := &unstructured.Unstructured{}
|
||||
obj.SetGroupVersionKind(r.gvk)
|
||||
return obj
|
||||
}
|
||||
|
||||
// NewList returns an empty list of Application objects
|
||||
func (r *REST) NewList() runtime.Object {
|
||||
return &appsv1alpha1.ApplicationList{}
|
||||
obj := &unstructured.Unstructured{}
|
||||
obj.SetGroupVersionKind(r.gvk.GroupVersion().WithKind(r.kindName + "List"))
|
||||
obj.Object["items"] = make([]interface{}, 0)
|
||||
return obj
|
||||
}
|
||||
|
||||
// Kind returns the resource kind used for API discovery
|
||||
|
||||
Reference in New Issue
Block a user