From ef6696cfd2f16908536ba3b129b3091b63f7daa5 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 11 Dec 2023 19:38:46 +0100 Subject: [PATCH] add some --- TODO | 2 + clusters/pve/Makefile | 6 +- clusters/pve/controlplane.yaml | 408 +++++++++++++++++++++++++ clusters/pve/patch-control-plane.yaml | 7 +- clusters/pve/patch.yaml | 13 +- clusters/pve/talosconfig | 12 + clusters/pve/worker.yaml | 422 ++++++++++++++++++++++++++ teststand/proxmox/nftables.conf | 2 + 8 files changed, 865 insertions(+), 7 deletions(-) create mode 100644 clusters/pve/controlplane.yaml create mode 100644 clusters/pve/talosconfig create mode 100644 clusters/pve/worker.yaml diff --git a/TODO b/TODO index aff6ad23..e7b25de2 100644 --- a/TODO +++ b/TODO @@ -8,3 +8,5 @@ grafana admin password grafana redis password autoconfigure ONCALL_API_URL oidc +rename vm, vm-longterm +talos setup via tcp-proxy diff --git a/clusters/pve/Makefile b/clusters/pve/Makefile index fc1e3ab2..4d6916ca 100644 --- a/clusters/pve/Makefile +++ b/clusters/pve/Makefile @@ -1,7 +1,7 @@ include ../../hack/app-talos.mk -export SERVER = https://192.168.0.110:6443 -export ENDPOINT := 192.168.0.110 -export NODES_CONTROL = 192.168.0.111 192.168.0.112 192.168.0.113 +export SERVER = https://192.168.100.10:6443 +export ENDPOINT := 135.181.169.168 +export NODES_CONTROL = 192.168.100.11 192.168.100.12 192.168.100.13 export NODES_WORKERS = export NODE := $(NODES_CONTROL) $(NODES_WORKERS) diff --git a/clusters/pve/controlplane.yaml b/clusters/pve/controlplane.yaml new file mode 100644 index 00000000..7e7c26f6 --- /dev/null +++ b/clusters/pve/controlplane.yaml @@ -0,0 +1,408 @@ +version: v1alpha1 +debug: false +persist: true +machine: + type: controlplane + token: e209sv.85mlwdix0ek04i89 + ca: + crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJQekNCOHFBREFnRUNBaEVBM0NoNEJsYmRUd2hhc0Roa0JFdHM1REFGQmdNclpYQXdFREVPTUF3R0ExVUUKQ2hNRmRHRnNiM013SGhjTk1qTXhNakE0TURreU56TXhXaGNOTXpNeE1qQTFNRGt5TnpNeFdqQVFNUTR3REFZRApWUVFLRXdWMFlXeHZjekFxTUFVR0F5dGxjQU1oQUR2UndSdWJrUm9WbmxmelZSZDNlWFh0b3VqY0hNK0dPTlJICm9zZDJzZmhsbzJFd1h6QU9CZ05WSFE4QkFmOEVCQU1DQW9Rd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSEF3RUcKQ0NzR0FRVUZCd01DTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRk9Tbm9IN3ZGMzN5Q0ZGVQpaR25yM0gzQ245N3JNQVVHQXl0bGNBTkJBRkNyRDZuV2VaRWtwcUJTVllwZ2hnUERmb2hiODlzQkp1VVhYR1diCit5R1NXZlExTFFXTlhOTUJOYjNSam9aMWxGTzhGc0lXYjN4SzhGSGJqM1diUWdvPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + key: LS0tLS1CRUdJTiBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0KTUM0Q0FRQXdCUVlESzJWd0JDSUVJQ0hKdWlhSDY5Zng2clNQdzF2YW0relZQb2oyK2RCODlaWFFTYnNrT09KVAotLS0tLUVORCBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0K + certSANs: + - 127.0.0.1 + - 135.181.169.168 + kubelet: + image: ghcr.io/siderolabs/kubelet:v1.29.0-rc.1 + defaultRuntimeSeccompProfileEnabled: true + nodeIP: + validSubnets: + - 192.168.100.0/24 + disableManifestsDirectory: true + + # clusterDNS: + # - 10.96.0.10 + # - 169.254.2.53 + + # extraArgs: + # key: value + + # extraMounts: + # - destination: /var/lib/example + # type: bind + # source: /var/lib/example + # options: + # - bind + # - rshared + # - rw + + # extraConfig: + # serverTLSBootstrap: true + + # credentialProviderConfig: + # apiVersion: kubelet.config.k8s.io/v1 + # kind: CredentialProviderConfig + # providers: + # - apiVersion: credentialprovider.kubelet.k8s.io/v1 + # defaultCacheDuration: 12h + # matchImages: + # - '*.dkr.ecr.*.amazonaws.com' + # - '*.dkr.ecr.*.amazonaws.com.cn' + # - '*.dkr.ecr-fips.*.amazonaws.com' + # - '*.dkr.ecr.us-iso-east-1.c2s.ic.gov' + # - '*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov' + # name: ecr-credential-provider + network: + interfaces: + - interface: eth0 + vip: + ip: 192.168.100.10 + + # # select a device with bus prefix 00:*. + # deviceSelector: + # busPath: 00:* + # # select a device with mac address matching `*:f0:ab` and `virtio` kernel driver. + # deviceSelector: + # hardwareAddr: '*:f0:ab' + # driver: virtio + # # select a device with bus prefix 00:*, a device with mac address matching `*:f0:ab` and `virtio` kernel driver. + # deviceSelector: + # - busPath: 00:* + # - hardwareAddr: '*:f0:ab' + # driver: virtio + + # addresses: + # - 10.5.0.0/16 + # - 192.168.3.7 + + # routes: + # - network: 0.0.0.0/0 + # gateway: 10.5.0.1 + # - network: 10.2.0.0/16 + # gateway: 10.2.0.1 + + # bond: + # interfaces: + # - enp2s0 + # - enp2s1 + # deviceSelectors: + # - busPath: 00:* + # - hardwareAddr: '*:f0:ab' + # driver: virtio + # mode: 802.3ad + # lacpRate: fast + + # bridge: + # interfaces: + # - enxda4042ca9a51 + # - enxae2a6774c259 + # stp: + # enabled: true + + # dhcp: true + + # dhcpOptions: + # routeMetric: 1024 + + # # wireguard server example + # wireguard: + # privateKey: ABCDEF... + # listenPort: 51111 + # peers: + # - publicKey: ABCDEF... + # endpoint: 192.168.1.3 + # allowedIPs: + # - 192.168.1.0/24 + # # wireguard peer example + # wireguard: + # privateKey: ABCDEF... + # peers: + # - publicKey: ABCDEF... + # endpoint: 192.168.1.2:51822 + # persistentKeepaliveInterval: 10s + # allowedIPs: + # - 192.168.1.0/24 + + # nameservers: + # - 8.8.8.8 + # - 1.1.1.1 + + # extraHostEntries: + # - ip: 192.168.1.100 + # aliases: + # - example + # - example.domain.tld + + # kubespan: + # enabled: true + install: + disk: /dev/sda + image: ghcr.io/siderolabs/installer:v1.6.0-beta.1 + extensions: + - image: ghcr.io/siderolabs/drbd:9.2.6-v1.6.0-beta.1 + wipe: false + + # diskSelector: + # size: 4GB + # model: WDC* + # busPath: /pci0000:00/0000:00:17.0/ata1/host0/target0:0:0/0:0:0:0 + + # extraKernelArgs: + # - talos.platform=metal + # - reboot=k + registries: {} + # mirrors: + # ghcr.io: + # endpoints: + # - https://registry.insecure + # - https://ghcr.io/v2/ + + # config: + # registry.insecure: + # tls: + # insecureSkipVerify: true + # + # # clientIdentity: + # # crt: LS0tIEVYQU1QTEUgQ0VSVElGSUNBVEUgLS0t + # # key: LS0tIEVYQU1QTEUgS0VZIC0tLQ== + # + # # auth: + # # username: username + # # password: password + + features: + rbac: true + stableHostname: true + apidCheckExtKeyUsage: true + diskQuotaSupport: true + kubePrism: + enabled: true + port: 7445 + + # kubernetesTalosAPIAccess: + # enabled: true + # allowedRoles: + # - os:reader + # allowedKubernetesNamespaces: + # - kube-system + kernel: + modules: + - name: drbd + parameters: + - usermode_helper=disabled + - name: openvswitch + + # # ControlPlane definition example. + # controlPlane: + # controllerManager: + # disabled: false + # scheduler: + # disabled: true + + # # nginx static pod. + # pods: + # - apiVersion: v1 + # kind: pod + # metadata: + # name: nginx + # spec: + # containers: + # - image: nginx + # name: nginx + + # # MachineDisks list example. + # disks: + # - device: /dev/sdb + # partitions: + # - mountpoint: /var/mnt/extra + # + # # # Human readable representation. + # # size: 100 MB + # # # Precise value in bytes. + # # size: 1073741824 + + # # MachineFiles usage example. + # files: + # - content: '...' + # permissions: 0o666 + # path: /tmp/file.txt + # op: append + + # # Environment variables definition examples. + # env: + # GRPC_GO_LOG_SEVERITY_LEVEL: info + # GRPC_GO_LOG_VERBOSITY_LEVEL: "99" + # https_proxy: http://SERVER:PORT/ + # env: + # GRPC_GO_LOG_SEVERITY_LEVEL: error + # https_proxy: https://USERNAME:PASSWORD@SERVER:PORT/ + # env: + # https_proxy: http://DOMAIN\USERNAME:PASSWORD@SERVER:PORT/ + + # # Example configuration for cloudflare ntp server. + # time: + # disabled: false + # servers: + # - time.cloudflare.com + # bootTimeout: 2m0s + + # # MachineSysctls usage example. + # sysctls: + # kernel.domainname: talos.dev + # net.ipv4.ip_forward: "0" + # net/ipv6/conf/eth0.100/disable_ipv6: "1" + + # # MachineSysfs usage example. + # sysfs: + # devices.system.cpu.cpu0.cpufreq.scaling_governor: performance + + # systemDiskEncryption: + # ephemeral: + # provider: luks2 + # keys: + # - nodeID: {} + # slot: 0 + # + # # kms: + # # endpoint: https://192.168.88.21:4443 + # + # # cipher: aes-xts-plain64 + + # # blockSize: 4096 + + # # options: + # # - no_read_workqueue + # # - no_write_workqueue + + # udev: + # rules: + # - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="44", MODE="0660" + + # logging: + # destinations: + # - endpoint: tcp://1.2.3.4:12345 + # format: json_lines + + # seccompProfiles: + # - name: audit.json + # value: + # defaultAction: SCMP_ACT_LOG + + # # node labels example. + # nodeLabels: + # exampleLabel: exampleLabelValue + + # # node taints example. + # nodeTaints: + # exampleTaint: exampleTaintValue:NoSchedule +cluster: + id: S0S7JTpj8Nptg11rGqqRpXLpfyEWkJzNGOJn3c-66P0= + secret: 8OUSrjySVui1E4fY2imMxqEQKq3djYefKK7qIRR+KvU= + controlPlane: + endpoint: https://192.168.100.10:6443 + clusterName: pve + network: + cni: + name: none + dnsDomain: cluster.local + podSubnets: + - 10.244.0.0/16 + serviceSubnets: + - 10.96.0.0/16 + token: 4atk0g.58oee7zml2uccpfx + secretboxEncryptionSecret: jUivpt4iWkvQ+55XfMdWe2DZHDk4i6+uSFDI+xvZL78= + ca: + crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVMrZ0F3SUJBZ0lRVnI4MnB1QzJuckRtNHlxVDcvUldZVEFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSXpNVEl3T0RBNU1qY3pNVm9YRFRNek1USXdOVEE1TWpjegpNVm93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCSkg0TEhGbVdvUDdaYjdldG5Ta0g4ZVBjZTZaWVhDTFl6aWFmZTR2UnFSdGJnOTNzOVNqZUJBYjJ4bzIKMXovdTZPY3ZzNWR5WDdldGJDNUdWRnE3c0dTallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVUwNW15QlVzSTJmLzZSUVlUd0lRYUNma3R2M3d3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnYWJCREZxR3EKVkR4VmlJN0E5M1ovczQ4aHhoNnJzQWNsaVgydUduS21vbHNDSVFDbTVSVHMrckQ4akxDQkF1Z2xzamNZMkZDcApoWjU5QzNIOFNOVy8zOFY3YXc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUdCTFZDNXUyTVNMTmJEelh2QkttdFpISWY2RWl1dWJtZG8wNldlWDY3K0RvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFa2Znc2NXWmFnL3RsdnQ2MmRLUWZ4NDl4N3BsaGNJdGpPSnA5N2k5R3BHMXVEM2V6MUtONApFQnZiR2piWFArN281eSt6bDNKZnQ2MXNMa1pVV3J1d1pBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= + aggregatorCA: + crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJZRENDQVFXZ0F3SUJBZ0lRWjQ0U2hpc0RxMi8xN1hCRE9NZzBFekFLQmdncWhrak9QUVFEQWpBQU1CNFgKRFRJek1USXdPREE1TWpjek1Wb1hEVE16TVRJd05UQTVNamN6TVZvd0FEQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxRwpTTTQ5QXdFSEEwSUFCTDNrSmFGem0zQ2dJYXMyemJiU1hRUmUyM0o0emZXRXRYWVpFUjJUUXdTL1Y1MktxN0lzCkhSeHg1Q09mOTBVakVnQXlsOExGODFiTVZNaUJnNUpTREhDallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWQKQmdOVkhTVUVGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZApCZ05WSFE0RUZnUVVhTG4zT2FmeFhBWVo4QVNXa3JYTVkzWDlMMUV3Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loCkFKR3BEUmNvVEZzK3AzZFlOS3BZQUNBUC9zbWlHaHJiU3k1Y0V6R2VDWHd4QWlFQW0xNkdXbEtmSk4rbms2aS8KYjBLZU5HUVUxV3dIWGpwbXJESGE0S0ZqVWZJPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSU96bEcwZlUvM0pwQXU1NVlvRVBKOE9BUk9mWVBqV1JYWGYvOW1vZ013M0lvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFdmVRbG9YT2JjS0FocXpiTnR0SmRCRjdiY25qTjlZUzFkaGtSSFpOREJMOVhuWXFyc2l3ZApISEhrSTUvM1JTTVNBREtYd3NYelZzeFV5SUdEa2xJTWNBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= + serviceAccount: + key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUFoM1pvOHQxdTEwWmVFRWp4VU5Qa2swdUZKckZIWVJ3ZGxqWmlXT1FraGVvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFNVEycVZhejA3eDN3OXlBMzc0VEhuTzFFelI1dDU1cVJzT1BOa2NiQUJnek8zQ1pkb1Q5awpCQWxtYWpYc0FtWFJCM2lwN2RYejB3VUtNQmVUMVNpMVlRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= + apiServer: + image: registry.k8s.io/kube-apiserver:v1.29.0-rc.1 + certSANs: + - 192.168.100.10 + - 127.0.0.1 + - 192.168.100.10 + - 135.181.169.168 + disablePodSecurityPolicy: true + admissionControl: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1alpha1 + defaults: + audit: restricted + audit-version: latest + enforce: baseline + enforce-version: latest + warn: restricted + warn-version: latest + exemptions: + namespaces: + - kube-system + runtimeClasses: [] + usernames: [] + kind: PodSecurityConfiguration + + # auditPolicy: + # apiVersion: audit.k8s.io/v1 + # kind: Policy + # rules: + # - level: Metadata + controllerManager: + image: registry.k8s.io/kube-controller-manager:v1.29.0-rc.1 + extraArgs: + bind-address: 0.0.0.0 + proxy: + disabled: true + image: registry.k8s.io/kube-proxy:v1.29.0-rc.1 + scheduler: + image: registry.k8s.io/kube-scheduler:v1.29.0-rc.1 + extraArgs: + bind-address: 0.0.0.0 + discovery: + enabled: false + registries: + kubernetes: + disabled: true + service: {} + # endpoint: https://discovery.talos.dev/ + etcd: + ca: + crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJmRENDQVNPZ0F3SUJBZ0lRTmRwc0cvTmpOZklLbzNDa1B1dmFNakFLQmdncWhrak9QUVFEQWpBUE1RMHcKQ3dZRFZRUUtFd1JsZEdOa01CNFhEVEl6TVRJd09EQTVNamN6TVZvWERUTXpNVEl3TlRBNU1qY3pNVm93RHpFTgpNQXNHQTFVRUNoTUVaWFJqWkRCWk1CTUdCeXFHU000OUFnRUdDQ3FHU000OUF3RUhBMElBQkEwRTdQU21jOWt3CitteW5tMDVvM3hIYkQ5RVdTdEpjbm12VE8wNXc2WUhMNUZPRlpEWEM4SlVwL3IxN2JTRUN5cmRTNzVnY3NuZnYKTG1ocmdIRFFBT2FqWVRCZk1BNEdBMVVkRHdFQi93UUVBd0lDaERBZEJnTlZIU1VFRmpBVUJnZ3JCZ0VGQlFjRApBUVlJS3dZQkJRVUhBd0l3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVSzA1VXphQVlIUU94CkVzRTJ1ZmdWcFF6UElwMHdDZ1lJS29aSXpqMEVBd0lEUndBd1JBSWdmdUJEWFFXUGJKZCtSV1FybUt3RldPYWYKcVBNcTZpb3FkTVBka3ZPazJaNENJRHUwbGcwZURDWUZOWW1kQ2t3eTUxYWE3bEM1cm4xbTgzMmxEWjdpdVR2VAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUMwMmswSUh5MTBLM1BqZWxpemxndjdxYlllT0FwdmdKR2Zod2JBaERNQkJvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFRFFUczlLWnoyVEQ2YktlYlRtamZFZHNQMFJaSzBseWVhOU03VG5EcGdjdmtVNFZrTmNMdwpsU24rdlh0dElRTEt0MUx2bUJ5eWQrOHVhR3VBY05BQTVnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= + advertisedSubnets: + - 192.168.100.0/24 + + # image: gcr.io/etcd-development/etcd:v3.5.11-arm64 + extraManifests: [] + # - https://www.example.com/manifest1.yaml + # - https://www.example.com/manifest2.yaml + + inlineManifests: [] + # - name: namespace-ci + # contents: |- + # apiVersion: v1 + # kind: Namespace + # metadata: + # name: ci + + allowSchedulingOnControlPlanes: true + + # # Decryption secret example (do not use in production!). + # aescbcEncryptionSecret: z01mye6j16bspJYtTB/5SFX8j7Ph4JXxM2Xuu4vsBPM= + + # coreDNS: + # image: registry.k8s.io/coredns/coredns:v1.11.1 + + # externalCloudProvider: + # enabled: true + # manifests: + # - https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/rbac.yaml + # - https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/aws-cloud-controller-manager-daemonset.yaml + + # extraManifestHeaders: + # Token: "1234567" + # X-ExtraInfo: info + + # adminKubeconfig: + # certLifetime: 1h0m0s diff --git a/clusters/pve/patch-control-plane.yaml b/clusters/pve/patch-control-plane.yaml index a69e727a..c63d5fb8 100644 --- a/clusters/pve/patch-control-plane.yaml +++ b/clusters/pve/patch-control-plane.yaml @@ -3,14 +3,15 @@ machine: interfaces: - interface: eth0 vip: - ip: 192.168.0.110 + ip: 192.168.100.10 cluster: allowSchedulingOnControlPlanes: true apiServer: certSANs: - 127.0.0.1 - - 192.168.0.101 + - 192.168.100.10 + - 135.181.169.168 controllerManager: extraArgs: bind-address: 0.0.0.0 @@ -25,4 +26,4 @@ cluster: enabled: false etcd: advertisedSubnets: - - 192.168.0.0/24 + - 192.168.100.0/24 diff --git a/clusters/pve/patch.yaml b/clusters/pve/patch.yaml index 5ddc2534..128c728c 100644 --- a/clusters/pve/patch.yaml +++ b/clusters/pve/patch.yaml @@ -1,14 +1,25 @@ machine: certSANs: - 127.0.0.1 + - 135.181.169.168 kubelet: nodeIP: validSubnets: - - 192.168.0.0/24 + - 192.168.100.0/24 kernel: modules: + - name: drbd + parameters: + - usermode_helper=disabled - name: openvswitch + install: + disk: /dev/sda + image: ghcr.io/siderolabs/installer:v1.6.0-beta.1 + wipe: false + + extensions: + - image: ghcr.io/siderolabs/drbd:9.2.6-v1.6.0-beta.1 cluster: network: diff --git a/clusters/pve/talosconfig b/clusters/pve/talosconfig new file mode 100644 index 00000000..909bb0be --- /dev/null +++ b/clusters/pve/talosconfig @@ -0,0 +1,12 @@ +context: pve +contexts: + pve: + endpoints: + - 135.181.169.168 + nodes: + - 192.168.100.11 + - 192.168.100.12 + - 192.168.100.13 + ca: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJQekNCOHFBREFnRUNBaEVBM0NoNEJsYmRUd2hhc0Roa0JFdHM1REFGQmdNclpYQXdFREVPTUF3R0ExVUUKQ2hNRmRHRnNiM013SGhjTk1qTXhNakE0TURreU56TXhXaGNOTXpNeE1qQTFNRGt5TnpNeFdqQVFNUTR3REFZRApWUVFLRXdWMFlXeHZjekFxTUFVR0F5dGxjQU1oQUR2UndSdWJrUm9WbmxmelZSZDNlWFh0b3VqY0hNK0dPTlJICm9zZDJzZmhsbzJFd1h6QU9CZ05WSFE4QkFmOEVCQU1DQW9Rd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSEF3RUcKQ0NzR0FRVUZCd01DTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRk9Tbm9IN3ZGMzN5Q0ZGVQpaR25yM0gzQ245N3JNQVVHQXl0bGNBTkJBRkNyRDZuV2VaRWtwcUJTVllwZ2hnUERmb2hiODlzQkp1VVhYR1diCit5R1NXZlExTFFXTlhOTUJOYjNSam9aMWxGTzhGc0lXYjN4SzhGSGJqM1diUWdvPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJLVENCM0tBREFnRUNBaEVBcHVGOWphUnlvTC9xVFhyVThkVzFzakFGQmdNclpYQXdFREVPTUF3R0ExVUUKQ2hNRmRHRnNiM013SGhjTk1qTXhNakV4TVRRMU1qRTRXaGNOTWpReE1qRXdNVFExTWpFNFdqQVRNUkV3RHdZRApWUVFLRXdodmN6cGhaRzFwYmpBcU1BVUdBeXRsY0FNaEFGcGlkOS93SjBDQzdHQkxSdll4Zzd5eG9kM3psN2RXCnZub01QRFFBTFY0aW8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0I0QXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0h3WURWUjBqQkJnd0ZvQVU1S2VnZnU4WGZmSUlVVlJrYWV2Y2ZjS2YzdXN3QlFZREsyVndBMEVBbTdLbQpVOC9OWXlqOVQ0VWR2VHNJeU1HWDZiQ25wczF2VDUzS1QzakRNSnB1VUJTSE5rMngxT25aWXlYb2hKbTNGQnVlCnc0NjNrVUhGTUhpeEhJeURCQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + key: LS0tLS1CRUdJTiBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0KTUM0Q0FRQXdCUVlESzJWd0JDSUVJT3VYOUtzZzJ0ckYwOUpCalJWeGw3Q2ZwV0hBRktTU1gydzNsK3lkbk5wMAotLS0tLUVORCBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0K diff --git a/clusters/pve/worker.yaml b/clusters/pve/worker.yaml new file mode 100644 index 00000000..1ea64865 --- /dev/null +++ b/clusters/pve/worker.yaml @@ -0,0 +1,422 @@ +version: v1alpha1 +debug: false +persist: true +machine: + type: worker + token: e209sv.85mlwdix0ek04i89 + ca: + crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJQekNCOHFBREFnRUNBaEVBM0NoNEJsYmRUd2hhc0Roa0JFdHM1REFGQmdNclpYQXdFREVPTUF3R0ExVUUKQ2hNRmRHRnNiM013SGhjTk1qTXhNakE0TURreU56TXhXaGNOTXpNeE1qQTFNRGt5TnpNeFdqQVFNUTR3REFZRApWUVFLRXdWMFlXeHZjekFxTUFVR0F5dGxjQU1oQUR2UndSdWJrUm9WbmxmelZSZDNlWFh0b3VqY0hNK0dPTlJICm9zZDJzZmhsbzJFd1h6QU9CZ05WSFE4QkFmOEVCQU1DQW9Rd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSEF3RUcKQ0NzR0FRVUZCd01DTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRk9Tbm9IN3ZGMzN5Q0ZGVQpaR25yM0gzQ245N3JNQVVHQXl0bGNBTkJBRkNyRDZuV2VaRWtwcUJTVllwZ2hnUERmb2hiODlzQkp1VVhYR1diCit5R1NXZlExTFFXTlhOTUJOYjNSam9aMWxGTzhGc0lXYjN4SzhGSGJqM1diUWdvPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + key: "" + certSANs: + - 127.0.0.1 + - 135.181.169.168 + kubelet: + image: ghcr.io/siderolabs/kubelet:v1.29.0-rc.1 + defaultRuntimeSeccompProfileEnabled: true + nodeIP: + validSubnets: + - 192.168.100.0/24 + disableManifestsDirectory: true + + # clusterDNS: + # - 10.96.0.10 + # - 169.254.2.53 + + # extraArgs: + # key: value + + # extraMounts: + # - destination: /var/lib/example + # type: bind + # source: /var/lib/example + # options: + # - bind + # - rshared + # - rw + + # extraConfig: + # serverTLSBootstrap: true + + # credentialProviderConfig: + # apiVersion: kubelet.config.k8s.io/v1 + # kind: CredentialProviderConfig + # providers: + # - apiVersion: credentialprovider.kubelet.k8s.io/v1 + # defaultCacheDuration: 12h + # matchImages: + # - '*.dkr.ecr.*.amazonaws.com' + # - '*.dkr.ecr.*.amazonaws.com.cn' + # - '*.dkr.ecr-fips.*.amazonaws.com' + # - '*.dkr.ecr.us-iso-east-1.c2s.ic.gov' + # - '*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov' + # name: ecr-credential-provider + network: {} + # interfaces: + # - interface: enp0s1 + # addresses: + # - 192.168.2.0/24 + # routes: + # - network: 0.0.0.0/0 + # gateway: 192.168.2.1 + # metric: 1024 + # mtu: 1500 + # + # # # select a device with bus prefix 00:*. + # # deviceSelector: + # # busPath: 00:* + # # # select a device with mac address matching `*:f0:ab` and `virtio` kernel driver. + # # deviceSelector: + # # hardwareAddr: '*:f0:ab' + # # driver: virtio + # # # select a device with bus prefix 00:*, a device with mac address matching `*:f0:ab` and `virtio` kernel driver. + # # deviceSelector: + # # - busPath: 00:* + # # - hardwareAddr: '*:f0:ab' + # # driver: virtio + + # # bond: + # # interfaces: + # # - enp2s0 + # # - enp2s1 + # # deviceSelectors: + # # - busPath: 00:* + # # - hardwareAddr: '*:f0:ab' + # # driver: virtio + # # mode: 802.3ad + # # lacpRate: fast + + # # bridge: + # # interfaces: + # # - enxda4042ca9a51 + # # - enxae2a6774c259 + # # stp: + # # enabled: true + + # # dhcp: true + + # # dhcpOptions: + # # routeMetric: 1024 + + # # # wireguard server example + # # wireguard: + # # privateKey: ABCDEF... + # # listenPort: 51111 + # # peers: + # # - publicKey: ABCDEF... + # # endpoint: 192.168.1.3 + # # allowedIPs: + # # - 192.168.1.0/24 + # # # wireguard peer example + # # wireguard: + # # privateKey: ABCDEF... + # # peers: + # # - publicKey: ABCDEF... + # # endpoint: 192.168.1.2:51822 + # # persistentKeepaliveInterval: 10s + # # allowedIPs: + # # - 192.168.1.0/24 + + # # # layer2 vip example + # # vip: + # # ip: 172.16.199.55 + + # nameservers: + # - 8.8.8.8 + # - 1.1.1.1 + + # extraHostEntries: + # - ip: 192.168.1.100 + # aliases: + # - example + # - example.domain.tld + + # kubespan: + # enabled: true + + install: + disk: /dev/sda + image: ghcr.io/siderolabs/installer:v1.6.0-beta.1 + extensions: + - image: ghcr.io/siderolabs/drbd:9.2.6-v1.6.0-beta.1 + wipe: false + + # diskSelector: + # size: 4GB + # model: WDC* + # busPath: /pci0000:00/0000:00:17.0/ata1/host0/target0:0:0/0:0:0:0 + + # extraKernelArgs: + # - talos.platform=metal + # - reboot=k + registries: {} + # mirrors: + # ghcr.io: + # endpoints: + # - https://registry.insecure + # - https://ghcr.io/v2/ + + # config: + # registry.insecure: + # tls: + # insecureSkipVerify: true + # + # # clientIdentity: + # # crt: LS0tIEVYQU1QTEUgQ0VSVElGSUNBVEUgLS0t + # # key: LS0tIEVYQU1QTEUgS0VZIC0tLQ== + # + # # auth: + # # username: username + # # password: password + + features: + rbac: true + stableHostname: true + apidCheckExtKeyUsage: true + diskQuotaSupport: true + kubePrism: + enabled: true + port: 7445 + + # kubernetesTalosAPIAccess: + # enabled: true + # allowedRoles: + # - os:reader + # allowedKubernetesNamespaces: + # - kube-system + kernel: + modules: + - name: drbd + parameters: + - usermode_helper=disabled + - name: openvswitch + + # # ControlPlane definition example. + # controlPlane: + # controllerManager: + # disabled: false + # scheduler: + # disabled: true + + # # nginx static pod. + # pods: + # - apiVersion: v1 + # kind: pod + # metadata: + # name: nginx + # spec: + # containers: + # - image: nginx + # name: nginx + + # # MachineDisks list example. + # disks: + # - device: /dev/sdb + # partitions: + # - mountpoint: /var/mnt/extra + # + # # # Human readable representation. + # # size: 100 MB + # # # Precise value in bytes. + # # size: 1073741824 + + # # MachineFiles usage example. + # files: + # - content: '...' + # permissions: 0o666 + # path: /tmp/file.txt + # op: append + + # # Environment variables definition examples. + # env: + # GRPC_GO_LOG_SEVERITY_LEVEL: info + # GRPC_GO_LOG_VERBOSITY_LEVEL: "99" + # https_proxy: http://SERVER:PORT/ + # env: + # GRPC_GO_LOG_SEVERITY_LEVEL: error + # https_proxy: https://USERNAME:PASSWORD@SERVER:PORT/ + # env: + # https_proxy: http://DOMAIN\USERNAME:PASSWORD@SERVER:PORT/ + + # # Example configuration for cloudflare ntp server. + # time: + # disabled: false + # servers: + # - time.cloudflare.com + # bootTimeout: 2m0s + + # # MachineSysctls usage example. + # sysctls: + # kernel.domainname: talos.dev + # net.ipv4.ip_forward: "0" + # net/ipv6/conf/eth0.100/disable_ipv6: "1" + + # # MachineSysfs usage example. + # sysfs: + # devices.system.cpu.cpu0.cpufreq.scaling_governor: performance + + # systemDiskEncryption: + # ephemeral: + # provider: luks2 + # keys: + # - nodeID: {} + # slot: 0 + # + # # kms: + # # endpoint: https://192.168.88.21:4443 + # + # # cipher: aes-xts-plain64 + + # # blockSize: 4096 + + # # options: + # # - no_read_workqueue + # # - no_write_workqueue + + # udev: + # rules: + # - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="44", MODE="0660" + + # logging: + # destinations: + # - endpoint: tcp://1.2.3.4:12345 + # format: json_lines + + # seccompProfiles: + # - name: audit.json + # value: + # defaultAction: SCMP_ACT_LOG + + # # node labels example. + # nodeLabels: + # exampleLabel: exampleLabelValue + + # # node taints example. + # nodeTaints: + # exampleTaint: exampleTaintValue:NoSchedule +cluster: + id: S0S7JTpj8Nptg11rGqqRpXLpfyEWkJzNGOJn3c-66P0= + secret: 8OUSrjySVui1E4fY2imMxqEQKq3djYefKK7qIRR+KvU= + controlPlane: + endpoint: https://192.168.100.10:6443 + network: + cni: + name: none + dnsDomain: cluster.local + podSubnets: + - 10.244.0.0/16 + serviceSubnets: + - 10.96.0.0/16 + token: 4atk0g.58oee7zml2uccpfx + ca: + crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVMrZ0F3SUJBZ0lRVnI4MnB1QzJuckRtNHlxVDcvUldZVEFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSXpNVEl3T0RBNU1qY3pNVm9YRFRNek1USXdOVEE1TWpjegpNVm93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCSkg0TEhGbVdvUDdaYjdldG5Ta0g4ZVBjZTZaWVhDTFl6aWFmZTR2UnFSdGJnOTNzOVNqZUJBYjJ4bzIKMXovdTZPY3ZzNWR5WDdldGJDNUdWRnE3c0dTallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVUwNW15QlVzSTJmLzZSUVlUd0lRYUNma3R2M3d3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnYWJCREZxR3EKVkR4VmlJN0E5M1ovczQ4aHhoNnJzQWNsaVgydUduS21vbHNDSVFDbTVSVHMrckQ4akxDQkF1Z2xzamNZMkZDcApoWjU5QzNIOFNOVy8zOFY3YXc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + key: "" + discovery: + enabled: true + registries: + kubernetes: + disabled: true + service: {} + # endpoint: https://discovery.talos.dev/ + + # # Decryption secret example (do not use in production!). + # aescbcEncryptionSecret: z01mye6j16bspJYtTB/5SFX8j7Ph4JXxM2Xuu4vsBPM= + + # # Decryption secret example (do not use in production!). + # secretboxEncryptionSecret: z01mye6j16bspJYtTB/5SFX8j7Ph4JXxM2Xuu4vsBPM= + + # # AggregatorCA example. + # aggregatorCA: + # crt: LS0tIEVYQU1QTEUgQ0VSVElGSUNBVEUgLS0t + # key: LS0tIEVYQU1QTEUgS0VZIC0tLQ== + + # # AggregatorCA example. + # serviceAccount: + # key: LS0tIEVYQU1QTEUgS0VZIC0tLQ== + + # apiServer: + # image: registry.k8s.io/kube-apiserver:v1.29.0-rc.1 + # extraArgs: + # feature-gates: ServerSideApply=true + # http2-max-streams-per-connection: "32" + # certSANs: + # - 1.2.3.4 + # - 4.5.6.7 + # admissionControl: + # - name: PodSecurity + # configuration: + # apiVersion: pod-security.admission.config.k8s.io/v1alpha1 + # defaults: + # audit: restricted + # audit-version: latest + # enforce: baseline + # enforce-version: latest + # warn: restricted + # warn-version: latest + # exemptions: + # namespaces: + # - kube-system + # runtimeClasses: [] + # usernames: [] + # kind: PodSecurityConfiguration + # auditPolicy: + # apiVersion: audit.k8s.io/v1 + # kind: Policy + # rules: + # - level: Metadata + + # controllerManager: + # image: registry.k8s.io/kube-controller-manager:v1.29.0-rc.1 + # extraArgs: + # feature-gates: ServerSideApply=true + + # proxy: + # disabled: false + # image: registry.k8s.io/kube-proxy:v1.29.0-rc.1 + # mode: ipvs + # extraArgs: + # proxy-mode: iptables + + # scheduler: + # image: registry.k8s.io/kube-scheduler:v1.29.0-rc.1 + # extraArgs: + # feature-gates: AllBeta=true + + # etcd: + # image: gcr.io/etcd-development/etcd:v3.5.11-arm64 + # ca: + # crt: LS0tIEVYQU1QTEUgQ0VSVElGSUNBVEUgLS0t + # key: LS0tIEVYQU1QTEUgS0VZIC0tLQ== + # extraArgs: + # election-timeout: "5000" + # advertisedSubnets: + # - 10.0.0.0/8 + + # coreDNS: + # image: registry.k8s.io/coredns/coredns:v1.11.1 + + # externalCloudProvider: + # enabled: true + # manifests: + # - https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/rbac.yaml + # - https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/aws-cloud-controller-manager-daemonset.yaml + + # extraManifests: + # - https://www.example.com/manifest1.yaml + # - https://www.example.com/manifest2.yaml + + # extraManifestHeaders: + # Token: "1234567" + # X-ExtraInfo: info + + # inlineManifests: + # - name: namespace-ci + # contents: |- + # apiVersion: v1 + # kind: Namespace + # metadata: + # name: ci + + # adminKubeconfig: + # certLifetime: 1h0m0s + + # allowSchedulingOnControlPlanes: true diff --git a/teststand/proxmox/nftables.conf b/teststand/proxmox/nftables.conf index fd076c28..9336a294 100644 --- a/teststand/proxmox/nftables.conf +++ b/teststand/proxmox/nftables.conf @@ -12,6 +12,8 @@ table inet filter { iif lo accept comment "accept loopback" + ip saddr 0.0.0.0/0 tcp dport 22 accept comment "accept ssh" + ip saddr 0.0.0.0/0 tcp dport 8006 accept comment "accept proxmox" ip saddr 0.0.0.0/0 tcp dport 6443 accept comment "accept kubernetes" ip saddr 0.0.0.0/0 tcp dport 5000 accept comment "accept talos"