diff --git a/oracle/Makefile b/oracle/Makefile index e312ca0..9d8e0c3 100644 --- a/oracle/Makefile +++ b/oracle/Makefile @@ -40,4 +40,7 @@ create-kubeconfig: talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.1.11 kubeconfig . create-deployments: - helm template --namespace=kube-system --version=1.11.0 -f deployments/cilium.yaml cilium cilium/cilium > deployments/cilium_result.yaml + helm template --namespace=kube-system --version=1.11.0 -f deployments/cilium.yaml cilium \ + cilium/cilium > deployments/cilium_result.yaml + helm template --namespace=ingress-nginx --version=4.0.16 -f deployments/ingress.yaml ingress-nginx \ + ingress-nginx/ingress-nginx > deployments/ingress_result.yaml diff --git a/oracle/deployments/ingress-ns.yaml b/oracle/deployments/ingress-ns.yaml new file mode 100644 index 0000000..6878f0b --- /dev/null +++ b/oracle/deployments/ingress-ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ingress-nginx diff --git a/oracle/deployments/ingress.yaml b/oracle/deployments/ingress.yaml new file mode 100644 index 0000000..6850f76 --- /dev/null +++ b/oracle/deployments/ingress.yaml @@ -0,0 +1,118 @@ + +controller: + kind: DaemonSet + + hostNetwork: true + hostPort: + enabled: false + ports: + http: 80 + https: 443 + + dnsPolicy: ClusterFirstWithHostNet + + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + + publishService: + enabled: false + + config: + worker-processes: "auto" + worker-cpu-affinity: "auto" + error-log-level: "error" + + server-tokens: "false" + http-redirect-code: "301" + + disable-ipv6-dns: "true" + + use-gzip: "true" + use-geoip: "false" + use-geoip2: "false" + + use-forwarded-headers: "true" + # curl https://www.cloudflare.com/ips-v4 2>/dev/null | tr '\n' ',' + proxy-real-ip-cidr: "173.245.48.0/20,103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,141.101.64.0/18,108.162.192.0/18,190.93.240.0/20,188.114.96.0/20,197.234.240.0/22,198.41.128.0/17,162.158.0.0/15,172.64.0.0/13,131.0.72.0/22,104.16.0.0/13,104.24.0.0/14,172.16.0.0/12" + + enable-access-log-for-default-backend: "true" + log-format-escape-json: "true" + log-format-upstream: '{"ip":"$remote_addr", "ssl":"$ssl_protocol", "method":"$request_method", "proto":"$scheme", "host":"$host", "uri":"$request_uri", "status":$status, "size":$bytes_sent, "agent":"$http_user_agent", "referer":"$http_referer", "namespace":"$namespace"}' + + upstream-keepalive-connections: "64" + proxy-connect-timeout: "10" + proxy-read-timeout: "120" + proxy-send-timeout: "120" + + ssl-protocols: "TLSv1.2 TLSv1.3" + + http-snippet: | + proxy_cache_path /tmp/static levels=1:2 use_temp_path=off keys_zone=static:16m inactive=24h max_size=512m; + + hsts: "true" + hsts-max-age: "31536000" + hsts-include-subdomains: "true" + hsts-preload: "true" + proxy-hide-headers: "strict-transport-security" + proxy-headers-hash-bucket-size: "128" + + server-name-hash-bucket-size: "64" + server-name-hash-max-size: "512" + + limit-req-status-code: "429" + + client-header-timeout: "30" + client-body-timeout: "30" + + minReadySeconds: 15 + + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" + + extraEnvs: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + periodSeconds: 30 + + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 100m + memory: 128Mi + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: project.io/node-pool + operator: In + values: + - web + + service: + enabled: true + type: ClusterIP + clusterIP: None + + admissionWebhooks: + enabled: false + metrics: + enabled: false + +revisionHistoryLimit: 2 + +defaultBackend: + enabled: false diff --git a/oracle/deployments/ingress_result.yaml b/oracle/deployments/ingress_result.yaml new file mode 100644 index 0000000..563aaaa --- /dev/null +++ b/oracle/deployments/ingress_result.yaml @@ -0,0 +1,440 @@ +--- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.1.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +automountServiceAccountToken: true +--- +# Source: ingress-nginx/templates/controller-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.1.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +data: + allow-snippet-annotations: "true" + client-body-timeout: "30" + client-header-timeout: "30" + disable-ipv6-dns: "true" + enable-access-log-for-default-backend: "true" + error-log-level: "error" + hsts: "true" + hsts-include-subdomains: "true" + hsts-max-age: "31536000" + hsts-preload: "true" + http-redirect-code: "301" + http-snippet: "proxy_cache_path /tmp/static levels=1:2 use_temp_path=off keys_zone=static:16m inactive=24h max_size=512m;\n" + limit-req-status-code: "429" + log-format-escape-json: "true" + log-format-upstream: "{\"ip\":\"$remote_addr\", \"ssl\":\"$ssl_protocol\", \"method\":\"$request_method\", \"proto\":\"$scheme\", \"host\":\"$host\", \"uri\":\"$request_uri\", \"status\":$status, \"size\":$bytes_sent, \"agent\":\"$http_user_agent\", \"referer\":\"$http_referer\", \"namespace\":\"$namespace\"}" + proxy-connect-timeout: "10" + proxy-headers-hash-bucket-size: "128" + proxy-hide-headers: "strict-transport-security" + proxy-read-timeout: "120" + proxy-real-ip-cidr: "173.245.48.0/20,103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,141.101.64.0/18,108.162.192.0/18,190.93.240.0/20,188.114.96.0/20,197.234.240.0/22,198.41.128.0/17,162.158.0.0/15,172.64.0.0/13,131.0.72.0/22,104.16.0.0/13,104.24.0.0/14,172.16.0.0/12" + proxy-send-timeout: "120" + server-name-hash-bucket-size: "64" + server-name-hash-max-size: "512" + server-tokens: "false" + ssl-protocols: "TLSv1.2 TLSv1.3" + upstream-keepalive-connections: "64" + use-forwarded-headers: "true" + use-geoip: "false" + use-geoip2: "false" + use-gzip: "true" + worker-cpu-affinity: "auto" + worker-processes: "auto" +--- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.1.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.1.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: "ingress-nginx" +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.1.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - ingress-controller-leader + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.1.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: "ingress-nginx" +--- +# Source: ingress-nginx/templates/controller-service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + labels: + helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.1.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + type: ClusterIP + clusterIP: None + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv4 + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + appProtocol: http + - name: https + port: 443 + protocol: TCP + targetPort: https + appProtocol: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.1.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + selector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 2 + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + minReadySeconds: 15 + template: + metadata: + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + spec: + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: controller + image: "k8s.gcr.io/ingress-nginx/controller:v1.1.1@sha256:0bc88eb15f9e7f84e8e56c14fa5735aaa488b840983f87bd79b1054190e660de" + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --election-id=ingress-controller-leader + - --controller-class=k8s.io/ingress-nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 100m + memory: 128Mi + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: project.io/node-pool + operator: In + values: + - web + serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: 300 +--- +# Source: ingress-nginx/templates/controller-ingressclass.yaml +# We don't support namespaced ingressClass yet +# So a ClusterRole and a ClusterRoleBinding is required +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.1.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: nginx +spec: + controller: k8s.io/ingress-nginx diff --git a/oracle/templates/controlplane.yaml.tpl b/oracle/templates/controlplane.yaml.tpl index 1e24ed9..fdbccda 100644 --- a/oracle/templates/controlplane.yaml.tpl +++ b/oracle/templates/controlplane.yaml.tpl @@ -77,3 +77,5 @@ cluster: - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/oracle/deployments/kubelet-serving-cert-approver.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/oracle/deployments/metrics-server.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/oracle/deployments/local-path-storage.yaml + - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/oracle/deployments/ingress-ns.yaml + - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/oracle/deployments/ingress_result.yaml