diff --git a/.renovate/groups.json5 b/.renovate/groups.json5 index 5cbc6fff..9456a3fb 100644 --- a/.renovate/groups.json5 +++ b/.renovate/groups.json5 @@ -28,6 +28,19 @@ "automerge": true, "automergeType": "pr" }, + // authentik + { + "description": "authentik Group", + "groupName": "authentik", + "matchPackagePatterns": ["authentik"], + "labels": ["authentik"] + "automerge": false, + "group": { + "commitMessageTopic": "{{{groupName}}} group" + }, + "separateMultipleMajor": true, + "separateMinorPatch": true + }, { "description": "Auto merge Github Actions", "matchManagers": ["github-actions"], diff --git a/.rtx.toml b/.rtx.toml index 9c267382..4ac62b5a 100644 --- a/.rtx.toml +++ b/.rtx.toml @@ -10,8 +10,8 @@ KUBECTL_COMMAND_HEADERS = "true" # kustomize = [""] # kubecolor = [""] flux2 = ["2.1.2"] -talosctl = ["1.5.4"] -talhelper = ["1.14.0"] +talosctl = ["1.5.4", "1.3.6"] +talhelper = ["1.16.2"] cilium-cli= ["0.15.14"] pulumi = ["3.95.0"] diff --git a/.taskfiles/pg/Taskfile.dist.yaml b/.taskfiles/pg/Taskfile.dist.yaml index 4ccefbe6..391c5d90 100644 --- a/.taskfiles/pg/Taskfile.dist.yaml +++ b/.taskfiles/pg/Taskfile.dist.yaml @@ -13,10 +13,10 @@ tasks: APP: "{{.APP}}" PG: &pg-default sh: |- - [[ -n "{{.PG}}" ]] && echo "{{.PG}}" || [[ -n "{{.APP}}" ]] && echo "pg-{{.APP}}" || echo "pg-default" + [[ -n "{{.PG}}" ]] && ( echo -n "{{.PG}}" && exit 0 ) || ( [[ -n "{{.APP}}" ]] && echo -n "pg-{{.APP}}" || echo -n "pg-default" ) NS: &ns-default sh: |- - [[ -n "{{.NS}}" ]] && echo "{{.NS}}" || [[ -n "{{.APP}}" ]] && echo "{{.APP}}" || echo "pg" + [[ -n "{{.NS}}" ]] && echo -n "{{.NS}}" || ( [[ -n "{{.APP}}" ]] && echo -n "{{.APP}}" || echo -n "pg" ) # PG: '{{ or .PG (fail "Missing `PG` environment variable!") }}' # NS: &ns-default '{{.NS | default "pg"}}' cmds: @@ -35,27 +35,42 @@ tasks: - | kubectl get clusters.postgresql.cnpg.io -n {{.NS}} {{.PG}} -o yaml | yq '.status.currentPrimary' | xargs -o -I% bash -c "while kubectl netshoot debug -n {{.NS}} % --image-name adminer --image-tag 4.8.1-standalone; do kubectl port-forward -n {{.NS}} % --address {{.PF_ADDR}} {{.PF_PORT}}:8080; break; done" - crunchy-restore: + crunchy-owner: desc: Restore a pg_dump to a CrunchyData Postgres cluster, writing to the master instance. dir: "{{.USER_WORKING_DIR}}" - vars: + vars: &crunchy-vars APP: "{{.APP}}" PG: *pg-default NS: *ns-default - DUMP: '{{ or .DUMP (fail "Missing `DUMP` environment variable for specifying pg_dump file location!") }}' - CP_DIR: '{{ .CP_DIR | default "/pgdata" }}' # would've been /tmp or /run if Crunchy replica container didn't crash and restart when files are copied there for some reason, maybe small tmpfs size? - DBNAME: - sh: |- - [[ -n "{{.DBNAME}}" ]] && echo "{{.DBNAME}}" || [[ -n "{{.APP}}" ]] && echo "{{.APP}}" || ( echo "Missing `DB` environment variable for specifying database name!" && exit 1 ) - DBUSER: - sh: |- - [[ -n "{{.DBUSER}}" ]] && echo "{{.DBUSER}}" || [[ -n "{{.APP}}" ]] && echo "{{.APP}}" || ( echo "Missing `USER` environment variable for specifying user name!" && exit 1 ) - ARGS: # TODO: (for other users) these are my personal defaults, please check and change accordingly if copying this task before running it! - sh: |- - [[ -n "{{.ARGS}}" ]] && echo "{{.ARGS}}" || echo "--verbose --format=c --clean --if-exists --no-owner --role {{.DBUSER}}" - CRUNCHY_PRIMARY: + CRUNCHY_PRIMARY: &crunchy-primary sh: |- kubectl get pods -n {{.NS}} -l postgres-operator.crunchydata.com/role=master,postgres-operator.crunchydata.com/cluster={{.PG}} -o jsonpath='{.items[0].metadata.name}' + DBNAME: &dbname + sh: |- + [[ -n "{{.DBNAME}}" ]] && echo -n "{{.DBNAME}}" || ( [[ -n "{{.APP}}" ]] && echo -n "{{.APP}}" ) || ( echo "Missing `DB` environment variable for specifying database name!" && exit 1 ) + DBUSER: &dbuser + sh: |- + [[ -n "{{.DBUSER}}" ]] && echo -n "{{.DBUSER}}" || [[ -n "{{.APP}}" ]] && echo -n "{{.APP}}" || ( echo "Missing `USER` environment variable for specifying user name!" && exit 1 ) + env: + DBUSER: '{{.DBUSER}}' + cmds: + - kubectl exec -it --container database -n {{.NS}} {{.CRUNCHY_PRIMARY}} -- /bin/bash -c 'echo "ALTER DATABASE \"{{.DBNAME}}\" OWNER TO \"{{.DBUSER}}\";" | psql' + + crunchy-restore: + desc: Restore a pg_dump to a CrunchyData Postgres cluster, writing to the master instance. + dir: "{{.USER_WORKING_DIR}}" + vars: &crunchy-vars + APP: "{{.APP}}" + PG: *pg-default + NS: *ns-default + CRUNCHY_PRIMARY: *crunchy-primary + DBNAME: *dbname + DBUSER: *dbuser + DUMP: &dump '{{ or .DUMP (fail "Missing `DUMP` environment variable for specifying pg_dump file location!") }}' + CP_DIR: &cpdir '{{ .CP_DIR | default "/pgdata" }}' # would've been /tmp or /run if Crunchy replica container didn't crash and restart when files are copied there for some reason, maybe small tmpfs size? + ARGS: # TODO: (for other users) these are my personal defaults, please check and change accordingly if copying this task before running it! + sh: |- + [[ -n "{{.ARGS}}" ]] && echo -n "{{.ARGS}}" || echo -n "--verbose --format=c --clean --if-exists --no-owner --role {{.DBUSER}}" env: DBUSER: '{{.DBUSER}}' cmds: @@ -64,3 +79,22 @@ tasks: kubectl exec -it --container database -n {{.NS}} {{.CRUNCHY_PRIMARY}} -- /bin/bash -c 'echo "ALTER DATABASE \"{{.DBNAME}}\" OWNER TO \"{{.DBUSER}}\";" | psql' - kubectl exec -it --container database -n {{.NS}} {{.CRUNCHY_PRIMARY}} -- /bin/bash -c 'pg_restore --dbname {{.DBNAME}} {{.ARGS}} {{.CP_DIR}}/restore-dump.psql' - defer: kubectl exec -it -n {{.NS}} {{.CRUNCHY_PRIMARY}} -- /bin/bash -c 'rm -rf {{.CP_DIR}}/restore-dump.psql' + + crunchy-dump: + desc: Run pg_dump on the master instance of a CrunchyData Postgres cluster. + dir: "{{.USER_WORKING_DIR}}" + vars: + APP: "{{.APP}}" + PG: *pg-default + NS: *ns-default + CRUNCHY_PRIMARY: *crunchy-primary + DUMP: *dump + CP_DIR: *cpdir + DBNAME: *dbname + DBUSER: *dbuser + ARGS: + sh: |- + [[ -n "{{.ARGS}}" ]] && echo "{{.ARGS}}" || echo "--verbose --format=c --clean --if-exists --no-owner" + cmds: + - kubectl exec -it --container database -n {{.NS}} {{.CRUNCHY_PRIMARY}} -- /bin/bash -c 'rm -rf {{.CP_DIR}}/dump.psql && pg_dump --dbname {{.DBNAME}} {{.ARGS}} --file {{.CP_DIR}}/dump.psql' + - kubectl cp --container database {{.NS}}/{{.CRUNCHY_PRIMARY}}:{{.CP_DIR}}/dump.psql {{.DUMP}} diff --git a/.vscode/settings.json b/.vscode/settings.json index 60dde505..c22a4f4c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -112,6 +112,7 @@ "editor.multiCursorModifier": "ctrlCmd", "terminal.integrated.tabs.focusMode": "singleClick", "editor.tabSize": 2, + "explorer.sortOrder": "mixed", // "editor.tokenColorCustomizations": { // "comments": "", // "textMateRules": [] diff --git a/kube/deploy/vm/_base/ks.yaml b/kube/deploy/vm/_base/ks.yaml deleted file mode 100644 index 8ad31bf5..00000000 --- a/kube/deploy/vm/_base/ks.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: zz-vm-1-kubevirt-base - namespace: flux-system -spec: - path: ./kube/deploy/vm/_base/app - dependsOn: - - name: zz-vm-1-kubevirt-app \ No newline at end of file diff --git a/kube/deploy/vm/_base/kustomization.yaml b/kube/deploy/vm/_base/kustomization.yaml deleted file mode 100644 index 70a77029..00000000 --- a/kube/deploy/vm/_base/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ks.yaml diff --git a/kube/templates/test/app/netpol.yaml b/kube/templates/test/app/netpol.yaml deleted file mode 100644 index c67af5c8..00000000 --- a/kube/templates/test/app/netpol.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: &app ${APPNAME} - namespace: *app -spec: - endpointSelector: {} - ingress: - # same namespace - - fromEndpoints: - - matchLabels: - io.kubernetes.pod.namespace: *app - # ingress controller - - fromEndpoints: - - matchLabels: - io.kubernetes.pod.namespace: ingress - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - toPorts: - - ports: - - port: "443" - protocol: TCP - - port: "443" - protocol: UDP - rules: - http: - - {} - egress: - # same namespace - - toEndpoints: - - matchLabels: - io.kubernetes.pod.namespace: *app diff --git a/kube/templates/test/app/pvc.yaml b/kube/templates/test/app/pvc.yaml deleted file mode 100644 index 783d0543..00000000 --- a/kube/templates/test/app/pvc.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: "${APPNAME}-data" - namespace: &app "${APPNAME}" - labels: - app.kubernetes.io/name: *app - app.kubernetes.io/instance: *app - snapshot.home.arpa/enabled: "true" -spec: - storageClassName: "file" - accessModes: ["ReadWriteMany"] - resources: - requests: - storage: "50Gi" - dataSourceRef: - apiGroup: "volsync.backube" - kind: "ReplicationDestination" - name: "${APPNAME}-data-bootstrap" diff --git a/kube/templates/test/app/volsync.yaml b/kube/templates/test/app/volsync.yaml deleted file mode 100644 index 5ef7ec71..00000000 --- a/kube/templates/test/app/volsync.yaml +++ /dev/null @@ -1,59 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: "${APPNAME}-data-r2-restic" - namespace: "${APPNAME}" -type: Opaque -stringData: - RESTIC_REPOSITORY: "${SECRET_VOLSYNC_R2_REPO}/${APPNAME}" - RESTIC_PASSWORD: "${SECRET_VOLSYNC_PASSWORD}" - AWS_ACCESS_KEY_ID: "${SECRET_VOLSYNC_R2_ID}" - AWS_SECRET_ACCESS_KEY: "${SECRET_VOLSYNC_R2_KEY}" ---- -apiVersion: volsync.backube/v1alpha1 -kind: ReplicationSource -metadata: - name: "${APPNAME}-data-r2-restic" - namespace: "${APPNAME}" -spec: - sourcePVC: "${APPNAME}-data" - trigger: - schedule: "0 22 * * *" # 6am GMT+8 - restic: - copyMethod: "Snapshot" - pruneIntervalDays: 14 - repository: "${APPNAME}-r2-restic" - cacheCapacity: "2Gi" - cacheStorageClassName: "local" - storageClassName: &sc "file" - volumeSnapshotClassName: "file" - moverSecurityContext: - runAsUser: &uid ${APP_UID_APPNAME} - runAsGroup: *uid - fsGroup: *uid - retain: - daily: 14 - within: 7d ---- -apiVersion: volsync.backube/v1alpha1 -kind: ReplicationDestination -metadata: - name: "${APPNAME}-data-r2-bootstrap" - namespace: "${APPNAME}" -spec: - trigger: - manual: "restore-once-bootstrap" - restic: - repository: "${APPNAME}-data-restic" - copyMethod: "Snapshot" - cacheCapacity: "2Gi" - cacheStorageClassName: "local" - storageClassName: "file" - volumeSnapshotClassName: "file" - capacity: "50Gi" - accessModes: ["ReadWriteMany"] - moverSecurityContext: - runAsUser: &uid ${APP_UID_APPNAME} - runAsGroup: *uid - fsGroup: *uid diff --git a/kube/templates/test/ks.yaml b/kube/templates/test/ks.yaml index d5241722..4271034c 100644 --- a/kube/templates/test/ks.yaml +++ b/kube/templates/test/ks.yaml @@ -7,6 +7,47 @@ metadata: spec: path: ./kube/deploy/apps/${APPNAME}/app dependsOn: + - name: thelounge-pvc +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: thelounge-pvc + namespace: flux-system +spec: + path: ./kube/deploy/core/storage/volsync/template + targetNamespace: "${APPNAME}" + dependsOn: + - name: 1-core-storage-volsync-app - name: 1-core-storage-rook-ceph-cluster - - name: 1-core-ingress-nginx-app - #- name: 1-core-storage-volsync-app \ No newline at end of file + postBuild: + substitute: + PVC: "${APPNAME}-data" + SIZE: "10Gi" + SC: "block" + RUID: &uid !!str | + ${APP_UID_APPNAME} + RGID: !!str | + ${APP_UID_APPNAME} + RFSG: !!str | + ${APP_UID_APPNAME} +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: ${APPNAME}-db + namespace: flux-system + labels: + prune.flux.home.arpa/enabled: "true" + db.home.arpa/pg: "pg-default" +spec: + path: ./kube/deploy/core/db/pg/clusters/template/pguser + targetNamespace: "pg" + dependsOn: + - name: 1-core-db-pg-clusters-default + - name: 1-core-secrets-es-k8s + postBuild: + substitute: + PG_NAME: "default" + PG_DB_USER: &app "${APPNAME}" + PG_APP_NS: *app