diff --git a/oracle/.gitignore b/oracle/.gitignore index 17b1704..a6ae36e 100644 --- a/oracle/.gitignore +++ b/oracle/.gitignore @@ -1,2 +1,5 @@ +_cfgs/ +templates/controlplane.yaml *.json *.qcow2 +talosctl* diff --git a/oracle/Makefile b/oracle/Makefile index e69de29..a7b06a0 100644 --- a/oracle/Makefile +++ b/oracle/Makefile @@ -0,0 +1,33 @@ + +ENDPOINT:=${shell terraform output -raw controlplane_endpoint 2>/dev/null} + +help: + @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +create-network: ## Create networks + cd prepare && terraform init && terraform apply -auto-approve + +create-lb: ## Create loadbalancer + terraform init + terraform apply -auto-approve -target=output.controlplane_endpoint + +create-config: ## Genereate talos configs + talosctl gen config --output-dir _cfgs --with-docs=false --with-examples=false talos-k8s-oracle https://${ENDPOINT}:6443 + +create-templates: + @yq ea -P '. as $$item ireduce ({}; . * $$item )' _cfgs/controlplane.yaml templates/controlplane.yaml.tpl > templates/controlplane.yaml + @echo 'podSubnets: "10.32.0.0/12,fd00:10:32::/102"' > _cfgs/tfstate.vars + @echo 'serviceSubnets: "10.200.0.0/22,fd40:10:200::/112"' >> _cfgs/tfstate.vars + @echo 'nodeSubnets: "172.16.0.0/12"' >> _cfgs/tfstate.vars + @echo 'apiDomain: api.cluster.local' >> _cfgs/tfstate.vars + @yq eval '.cluster.network.dnsDomain' _cfgs/controlplane.yaml | awk '{ print "domain: "$$1}' >> _cfgs/tfstate.vars + @yq eval '.cluster.clusterName' _cfgs/controlplane.yaml | awk '{ print "clusterName: "$$1}' >> _cfgs/tfstate.vars + @yq eval '.machine.token' _cfgs/controlplane.yaml | awk '{ print "tokenMachine: "$$1}' >> _cfgs/tfstate.vars + @yq eval '.machine.ca.crt' _cfgs/controlplane.yaml | awk '{ print "caMachine: "$$1}' >> _cfgs/tfstate.vars + @yq eval '.cluster.token' _cfgs/controlplane.yaml | awk '{ print "token: "$$1}' >> _cfgs/tfstate.vars + @yq eval '.cluster.ca.crt' _cfgs/controlplane.yaml | awk '{ print "ca: "$$1}' >> _cfgs/tfstate.vars + + @yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.json + +create-kubeconfig: + talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.1.11 kubeconfig diff --git a/oracle/common.tf b/oracle/common.tf index eb49a72..b745e70 100644 --- a/oracle/common.tf +++ b/oracle/common.tf @@ -1,32 +1,35 @@ data "oci_core_images" "talos_x64" { compartment_id = var.compartment_ocid - operating_system = "Canonical Ubuntu" - # operating_system_version = "20.04" - state = "AVAILABLE" - sort_by = "TIMECREATED" + operating_system = "Talos" + state = "AVAILABLE" + sort_by = "TIMECREATED" - # filter { - # name = "launch_mode" - # values = ["NATIVE"] - # regex = true - # } - # filter { - # name = "display_name" - # values = ["Linux"] - # regex = true - # } - # filter { - # name = "network_type" - # values = ["VFIO"] - # } + filter { + name = "display_name" + values = ["amd64"] + regex = true + } } -data "oci_core_image_shapes" "talos_x64" { - image_id = data.oci_core_images.talos_x64.images[0].id +data "oci_core_images" "talos_arm" { + compartment_id = var.compartment_ocid + operating_system = "Talos" + state = "AVAILABLE" + sort_by = "TIMECREATED" + + filter { + name = "display_name" + values = ["arm64"] + regex = true + } } -data "oci_identity_fault_domains" "fault_domains" { +# data "oci_core_image_shapes" "talos_x64" { +# image_id = data.oci_core_images.talos_x64.images[0].id +# } + +data "oci_identity_fault_domains" "domains" { compartment_id = var.compartment_ocid - availability_domain = local.network_public["jNdv:eu-amsterdam-1-AD-1"].availability_domain + availability_domain = local.network_public[local.zone].availability_domain } diff --git a/oracle/instances-controlplane.tf b/oracle/instances-controlplane.tf index 914b767..dd4d546 100644 --- a/oracle/instances-controlplane.tf +++ b/oracle/instances-controlplane.tf @@ -1,68 +1,103 @@ -# data "oci_core_vnic_attachments" "contolplane" { -# compartment_id = var.compartment_ocid -# instance_id = oci_core_instance.contolplane.id -# } +data "oci_core_vnic_attachments" "contolplane" { + count = lookup(var.controlplane, "count", 0) + compartment_id = var.compartment_ocid + instance_id = oci_core_instance.contolplane[count.index].id +} -# resource "oci_core_ipv6" "contolplane" { -# vnic_id = data.oci_core_vnic_attachments.contolplane.vnic_attachments[0]["vnic_id"] -# } +resource "oci_core_ipv6" "contolplane" { + count = lookup(var.controlplane, "count", 0) + vnic_id = data.oci_core_vnic_attachments.contolplane[count.index].vnic_attachments[0]["vnic_id"] +} -# resource "oci_core_instance" "contolplane" { -# compartment_id = var.compartment_ocid -# display_name = "contolplane-1" -# availability_domain = local.network_public["jNdv:eu-amsterdam-1-AD-1"].availability_domain -# shape = "VM.Standard.E2.1.Micro" +resource "oci_core_instance" "contolplane" { + count = lookup(var.controlplane, "count", 0) -# metadata = { -# ssh_authorized_keys = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDd+wfWIKi1dDZuCsd/zNw2n4WuHHa21N/Ltmo3umH2d local" -# user_data = base64encode("# noop") -# } + compartment_id = var.compartment_ocid + display_name = "${local.project}-contolplane-${count.index + 1}" + availability_domain = local.zone + fault_domain = element(data.oci_identity_fault_domains.domains.fault_domains, count.index).name -# source_details { -# source_type = "image" -# source_id = data.oci_core_images.talos_x64.images[0].id -# boot_volume_size_in_gbs = "50" -# } -# create_vnic_details { -# assign_public_ip = true -# subnet_id = local.network_public["jNdv:eu-amsterdam-1-AD-1"].id -# private_ip = cidrhost(local.network_public["jNdv:eu-amsterdam-1-AD-1"].cidr_block, 11) -# nsg_ids = [local.nsg_talos, local.nsg_cilium] -# } + shape = lookup(var.controlplane, "type", "VM.Standard.E4.Flex") + shape_config { + ocpus = lookup(var.controlplane, "ocpus", 1) + memory_in_gbs = lookup(var.controlplane, "memgb", 3) + } -# launch_options { -# firmware = "UEFI_64" -# is_pv_encryption_in_transit_enabled = true -# remote_data_volume_type = "PARAVIRTUALIZED" -# network_type = "PARAVIRTUALIZED" -# } -# instance_options { -# are_legacy_imds_endpoints_disabled = true -# } -# availability_config { -# is_live_migration_preferred = true -# recovery_action = "RESTORE_INSTANCE" -# } + metadata = { + ssh_authorized_keys = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDd+wfWIKi1dDZuCsd/zNw2n4WuHHa21N/Ltmo3umH2d local" + user_data = base64encode(templatefile("${path.module}/templates/controlplane.yaml", + merge(var.kubernetes, { + name = "contolplane-${count.index + 1}" + lbv4 = local.lbv4 + lbv4_local = local.lbv4_local + nodeSubnets = local.network_public[local.zone].cidr_block + }) + )) + } -# timeouts { -# create = "10m" -# } + source_details { + source_type = "image" + source_id = data.oci_core_images.talos_x64.images[0].id + boot_volume_size_in_gbs = "50" + } + create_vnic_details { + assign_public_ip = true + subnet_id = local.network_public[local.zone].id + private_ip = cidrhost(local.network_public[local.zone].cidr_block, 11 + count.index) + nsg_ids = [local.nsg_talos, local.nsg_cilium, local.nsg_contolplane] + } -# lifecycle { -# ignore_changes = [ -# defined_tags, -# create_vnic_details["defined_tags"], -# launch_options["is_pv_encryption_in_transit_enabled"] -# ] -# } -# } + agent_config { + are_all_plugins_disabled = true + is_management_disabled = true + is_monitoring_disabled = true -# resource "oci_network_load_balancer_backend" "contolplane" { -# backend_set_name = oci_network_load_balancer_backend_set.contolplane.name -# network_load_balancer_id = oci_network_load_balancer_network_load_balancer.contolplane.id -# port = 80 + } + availability_config { + is_live_migration_preferred = true + recovery_action = "RESTORE_INSTANCE" + } + launch_options { + firmware = "UEFI_64" + boot_volume_type = "PARAVIRTUALIZED" + remote_data_volume_type = "PARAVIRTUALIZED" + network_type = "PARAVIRTUALIZED" + } + instance_options { + are_legacy_imds_endpoints_disabled = true + } -# name = "contolplane-1" -# target_id = oci_core_instance.contolplane.id -# } + timeouts { + create = "10m" + } + + lifecycle { + ignore_changes = [ + shape_config, + defined_tags, + create_vnic_details["defined_tags"], + launch_options["is_pv_encryption_in_transit_enabled"] + ] + } +} + +resource "oci_network_load_balancer_backend" "contolplane" { + count = local.lbv4_enable ? lookup(var.controlplane, "count", 0) : 0 + backend_set_name = oci_network_load_balancer_backend_set.contolplane[0].name + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.contolplane[0].id + port = 6443 + + name = "${local.project}-contolplane-${count.index + 1}" + target_id = oci_core_instance.contolplane[count.index].id +} + +resource "oci_network_load_balancer_backend" "contolplane_talos" { + count = local.lbv4_enable ? lookup(var.controlplane, "count", 0) : 0 + backend_set_name = oci_network_load_balancer_backend_set.contolplane_talos[0].name + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.contolplane[0].id + port = 50000 + + name = "${local.project}-contolplane-talos-${count.index + 1}" + target_id = oci_core_instance.contolplane[count.index].id +} diff --git a/oracle/instances-web.tf b/oracle/instances-web.tf index ef7f134..8cf2b04 100644 --- a/oracle/instances-web.tf +++ b/oracle/instances-web.tf @@ -1,117 +1,129 @@ -# resource "oci_core_instance_pool" "web" { -# compartment_id = var.compartment_ocid -# instance_configuration_id = oci_core_instance_configuration.web.id -# size = 0 -# state = "RUNNING" -# display_name = "${var.project}-web" +resource "oci_core_instance_pool" "web" { + compartment_id = var.compartment_ocid + instance_configuration_id = oci_core_instance_configuration.web.id + size = lookup(var.instances[local.zone], "web_count", 0) + state = "RUNNING" + display_name = "${var.project}-web" -# placement_configurations { -# availability_domain = local.network_public["jNdv:eu-amsterdam-1-AD-1"].availability_domain -# fault_domains = data.oci_identity_fault_domains.fault_domains.fault_domains.*.name -# primary_subnet_id = local.network_public["jNdv:eu-amsterdam-1-AD-1"].id -# } + placement_configurations { + availability_domain = local.network_public[local.zone].availability_domain + fault_domains = data.oci_identity_fault_domains.domains.fault_domains.*.name + primary_subnet_id = local.network_public[local.zone].id + } -# # load_balancers { -# # backend_set_name = oci_network_load_balancer_backend_set.web.name -# # load_balancer_id = oci_network_load_balancer_network_load_balancer.web.id -# # port = 80 -# # vnic_selection = "PrimaryVnic" -# # } + # load_balancers { + # backend_set_name = oci_network_load_balancer_backend_set.web.name + # load_balancer_id = oci_network_load_balancer_network_load_balancer.web.id + # port = 80 + # vnic_selection = "PrimaryVnic" + # } -# lifecycle { -# ignore_changes = [ -# size, -# state, -# defined_tags -# ] -# } -# } + lifecycle { + ignore_changes = [ + # size, + state, + defined_tags + ] + } +} -# resource "oci_core_instance_configuration" "web" { -# compartment_id = var.compartment_ocid -# display_name = "${var.project}-web" +resource "oci_core_instance_configuration" "web" { + compartment_id = var.compartment_ocid + display_name = "${var.project}-web" -# instance_details { -# instance_type = "compute" + instance_details { + instance_type = "compute" -# launch_details { -# compartment_id = var.compartment_ocid -# shape = "VM.Standard.E2.1.Micro" -# display_name = "${var.project}-web" -# is_pv_encryption_in_transit_enabled = true -# preferred_maintenance_action = "LIVE_MIGRATE" -# launch_mode = "NATIVE" + launch_details { + compartment_id = var.compartment_ocid + display_name = "${var.project}-web" + is_pv_encryption_in_transit_enabled = true + preferred_maintenance_action = "LIVE_MIGRATE" + launch_mode = "NATIVE" -# metadata = { -# ssh_authorized_keys = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDd+wfWIKi1dDZuCsd/zNw2n4WuHHa21N/Ltmo3umH2d local" -# } + shape = lookup(var.instances[local.zone], "web_instance_shape", "VM.Standard.E2.1.Micro") + shape_config { + ocpus = lookup(var.instances[local.zone], "web_instance_ocpus", 1) + memory_in_gbs = lookup(var.instances[local.zone], "web_instance_memgb", 1) + } -# source_details { -# source_type = "image" -# image_id = data.oci_core_images.talos_x64.images[0].id -# boot_volume_size_in_gbs = "50" -# } -# create_vnic_details { -# display_name = "${var.project}-web" -# assign_private_dns_record = false -# assign_public_ip = true -# nsg_ids = [local.nsg_talos, local.nsg_cilium, local.nsg_web] -# subnet_id = local.network_public["jNdv:eu-amsterdam-1-AD-1"].id -# } + metadata = { + user_data = templatefile("${path.module}/templates/web.yaml.tpl", + merge(var.kubernetes, { + lbv4 = local.lbv4_local + lbv4_web = local.lbv4_web + nodeSubnets = local.network_public[local.zone].cidr_block + }) + ) + } -# agent_config { -# is_management_disabled = false -# is_monitoring_disabled = false -# } -# launch_options { -# network_type = "PARAVIRTUALIZED" -# } -# instance_options { -# are_legacy_imds_endpoints_disabled = true -# } -# availability_config { -# recovery_action = "RESTORE_INSTANCE" -# } -# } -# } + source_details { + source_type = "image" + image_id = data.oci_core_images.talos_x64.images[0].id + boot_volume_size_in_gbs = "50" + } + create_vnic_details { + display_name = "${var.project}-web" + assign_private_dns_record = false + assign_public_ip = true + nsg_ids = [local.nsg_talos, local.nsg_cilium, local.nsg_web, local.nsg_contolplane] + subnet_id = local.network_public[local.zone].id + skip_source_dest_check = true + } -# lifecycle { -# create_before_destroy = "true" -# } -# } + agent_config { + is_management_disabled = false + is_monitoring_disabled = false + } + launch_options { + network_type = "PARAVIRTUALIZED" + } + instance_options { + are_legacy_imds_endpoints_disabled = true + } + availability_config { + recovery_action = "RESTORE_INSTANCE" + } + } + } -# data "oci_core_instance_pool_instances" "web" { -# compartment_id = var.compartment_ocid -# instance_pool_id = oci_core_instance_pool.web.id -# } + lifecycle { + create_before_destroy = "true" + } +} -# resource "oci_network_load_balancer_backend" "web_http" { -# for_each = { for instances in data.oci_core_instance_pool_instances.web.instances.* : instances.display_name => instances.id } +data "oci_core_instance_pool_instances" "web" { + compartment_id = var.compartment_ocid + instance_pool_id = oci_core_instance_pool.web.id +} -# backend_set_name = oci_network_load_balancer_backend_set.web_http.name -# network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web.id -# port = 80 +resource "oci_network_load_balancer_backend" "web_http" { + for_each = { for instances in data.oci_core_instance_pool_instances.web.instances.* : instances.display_name => instances.id } -# name = "web-http-lb" -# target_id = each.value + backend_set_name = oci_network_load_balancer_backend_set.web_http[0].name + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web[0].id + port = 80 -# depends_on = [ -# oci_core_instance_pool.web -# ] -# } + name = "web-http-lb" + target_id = each.value -# resource "oci_network_load_balancer_backend" "web_https" { -# for_each = { for instances in data.oci_core_instance_pool_instances.web.instances.* : instances.display_name => instances.id } + depends_on = [ + oci_core_instance_pool.web + ] +} -# backend_set_name = oci_network_load_balancer_backend_set.web_https.name -# network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web.id -# port = 443 +resource "oci_network_load_balancer_backend" "web_https" { + for_each = { for instances in data.oci_core_instance_pool_instances.web.instances.* : instances.display_name => instances.id } -# name = "web-https-lb" -# target_id = each.value + backend_set_name = oci_network_load_balancer_backend_set.web_https[0].name + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web[0].id + port = 443 -# depends_on = [ -# oci_core_instance_pool.web -# ] -# } + name = "web-https-lb" + target_id = each.value + + depends_on = [ + oci_core_instance_pool.web + ] +} diff --git a/oracle/network-lb.tf b/oracle/network-lb.tf index 39bf13e..8e821d7 100644 --- a/oracle/network-lb.tf +++ b/oracle/network-lb.tf @@ -1,91 +1,132 @@ -# resource "oci_network_load_balancer_network_load_balancer" "contolplane" { -# compartment_id = var.compartment_ocid -# display_name = "${local.project}-contolplane-lb" -# subnet_id = local.network_lb.id -# network_security_group_ids = [local.nsg_contolplane_lb] +locals { + lbv4_enable = false + lbv4 = local.lbv4_enable ? [for ip in oci_network_load_balancer_network_load_balancer.contolplane[0].ip_addresses : ip.ip_address if ip.is_public][0] : "127.0.0.1" + lbv4_local = local.lbv4_enable ? [for ip in oci_network_load_balancer_network_load_balancer.contolplane[0].ip_addresses : ip.ip_address if !ip.is_public][0] : "127.0.0.1" -# is_preserve_source_destination = false -# is_private = true -# } + lbv4_web_enable = false + lbv4_web = local.lbv4_web_enable ? [for ip in oci_network_load_balancer_network_load_balancer.web[0].ip_addresses : ip.ip_address if ip.is_public][0] : "127.0.0.1" +} -# resource "oci_network_load_balancer_listener" "contolplane" { -# default_backend_set_name = oci_network_load_balancer_backend_set.contolplane.name +resource "oci_network_load_balancer_network_load_balancer" "contolplane" { + count = local.lbv4_enable ? 1 : 0 + compartment_id = var.compartment_ocid + display_name = "${local.project}-contolplane-lb" + subnet_id = local.network_lb.id + network_security_group_ids = [local.nsg_contolplane_lb] -# name = "${local.project}-contolplane" -# network_load_balancer_id = oci_network_load_balancer_network_load_balancer.contolplane.id -# port = 80 -# protocol = "TCP" -# } + is_preserve_source_destination = false + is_private = false +} -# resource "oci_network_load_balancer_backend_set" "contolplane" { -# name = "${local.project}-contolplane" -# network_load_balancer_id = oci_network_load_balancer_network_load_balancer.contolplane.id -# policy = "FIVE_TUPLE" -# is_preserve_source = false +resource "oci_network_load_balancer_listener" "contolplane" { + count = local.lbv4_enable ? 1 : 0 + default_backend_set_name = oci_network_load_balancer_backend_set.contolplane[0].name -# health_checker { -# protocol = "HTTP" -# port = 80 -# url_path = "/" -# return_code = 200 -# } -# } + name = "${local.project}-contolplane" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.contolplane[0].id + port = 6443 + protocol = "TCP" +} +resource "oci_network_load_balancer_listener" "contolplane_talos" { + count = local.lbv4_enable ? 1 : 0 + default_backend_set_name = oci_network_load_balancer_backend_set.contolplane_talos[0].name -# resource "oci_network_load_balancer_network_load_balancer" "web" { -# compartment_id = var.compartment_ocid -# display_name = "${local.project}-web-lb" -# subnet_id = local.network_lb.id -# network_security_group_ids = [local.nsg_web] + name = "${local.project}-contolplane-talos" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.contolplane[0].id + port = 50000 + protocol = "TCP" +} -# is_preserve_source_destination = false -# is_private = false -# } +resource "oci_network_load_balancer_backend_set" "contolplane" { + count = local.lbv4_enable ? 1 : 0 + name = "${local.project}-contolplane" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.contolplane[0].id + policy = "FIVE_TUPLE" + is_preserve_source = false -# resource "oci_network_load_balancer_listener" "http" { -# default_backend_set_name = oci_network_load_balancer_backend_set.web_http.name + health_checker { + protocol = "HTTPS" + port = 6443 + url_path = "/readyz" + return_code = 200 + interval_in_millis = 15000 + } +} +resource "oci_network_load_balancer_backend_set" "contolplane_talos" { + count = local.lbv4_enable ? 1 : 0 + name = "${local.project}-contolplane-talos" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.contolplane[0].id + policy = "FIVE_TUPLE" + is_preserve_source = false -# name = "${local.project}-web-http" -# network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web.id -# port = 80 -# protocol = "TCP" -# } + health_checker { + protocol = "TCP" + port = 50000 + interval_in_millis = 30000 + } +} -# resource "oci_network_load_balancer_listener" "https" { -# default_backend_set_name = oci_network_load_balancer_backend_set.web_https.name +resource "oci_network_load_balancer_network_load_balancer" "web" { + count = local.lbv4_web_enable ? 1 : 0 + compartment_id = var.compartment_ocid + display_name = "${local.project}-web-lb" + subnet_id = local.network_lb.id + network_security_group_ids = [local.nsg_web] -# name = "${local.project}-web-https" -# network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web.id -# port = 443 -# protocol = "TCP" -# } + is_preserve_source_destination = true + is_private = false +} -# resource "oci_network_load_balancer_backend_set" "web_http" { -# name = "${local.project}-web-http" -# network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web.id -# policy = "FIVE_TUPLE" -# is_preserve_source = true +resource "oci_network_load_balancer_listener" "http" { + count = local.lbv4_web_enable ? 1 : 0 + default_backend_set_name = oci_network_load_balancer_backend_set.web_http[0].name -# health_checker { -# interval_in_millis = 30000 -# protocol = "HTTP" -# port = 80 -# url_path = "/" -# return_code = 200 -# } -# } + name = "${local.project}-web-http" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web[0].id + port = 80 + protocol = "TCP" +} -# resource "oci_network_load_balancer_backend_set" "web_https" { -# name = "${local.project}-web-https" -# network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web.id -# policy = "FIVE_TUPLE" -# is_preserve_source = true +resource "oci_network_load_balancer_backend_set" "web_http" { + count = local.lbv4_web_enable ? 1 : 0 + name = "${local.project}-web-http" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web[0].id + policy = "FIVE_TUPLE" + is_preserve_source = true -# health_checker { -# interval_in_millis = 30000 -# protocol = "HTTP" -# port = 80 -# url_path = "/" -# return_code = 200 -# } -# } + health_checker { + retries = 2 + interval_in_millis = 15000 + protocol = "HTTP" + port = 80 + url_path = "/" + return_code = 200 + } +} + +resource "oci_network_load_balancer_listener" "https" { + count = local.lbv4_web_enable ? 1 : 0 + default_backend_set_name = oci_network_load_balancer_backend_set.web_https[0].name + + name = "${local.project}-web-https" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web[0].id + port = 443 + protocol = "TCP" +} + +resource "oci_network_load_balancer_backend_set" "web_https" { + count = local.lbv4_web_enable ? 1 : 0 + name = "${local.project}-web-https" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.web[0].id + policy = "FIVE_TUPLE" + is_preserve_source = true + + health_checker { + interval_in_millis = 15000 + protocol = "HTTP" + port = 80 + url_path = "/" + return_code = 200 + } +} diff --git a/oracle/output.tf b/oracle/output.tf new file mode 100644 index 0000000..f898abe --- /dev/null +++ b/oracle/output.tf @@ -0,0 +1,10 @@ + +output "controlplane_endpoint" { + description = "Kubernetes controlplane endpoint" + value = local.lbv4 +} + +output "web_endpoint" { + description = "Web endpoint" + value = local.lbv4_web +} diff --git a/oracle/prepare/images.tf b/oracle/prepare/images.tf new file mode 100644 index 0000000..52ad448 --- /dev/null +++ b/oracle/prepare/images.tf @@ -0,0 +1,97 @@ + +resource "oci_objectstorage_object" "talos_amd64" { + bucket = oci_objectstorage_bucket.images.name + namespace = data.oci_objectstorage_namespace.ns.namespace + object = "talos-amd64.qcow2" + source = "oracle-amd64.qcow2" + content_md5 = filemd5("oracle-amd64.qcow2") +} + +resource "oci_core_image" "talos_amd64" { + compartment_id = var.tenancy_ocid + + display_name = "Talos-amd64" + launch_mode = "NATIVE" + + image_source_details { + source_type = "objectStorageTuple" + namespace_name = oci_objectstorage_bucket.images.namespace + bucket_name = oci_objectstorage_bucket.images.name + object_name = oci_objectstorage_object.talos_amd64.object + + operating_system = "Talos" + operating_system_version = "0.14.0" + source_image_type = "QCOW2" + } + + timeouts { + create = "30m" + } +} + +# resource "oci_core_compute_image_capability_schema" "talos_amd64" { +# compartment_id = var.tenancy_ocid + +# compute_global_image_capability_schema_version_name = data.oci_core_compute_global_image_capability_schemas_version.default.name + +# display_name = "Talos-amd64" +# image_id = oci_core_image.talos_amd64.id + +# schema_data = { +# "Storage.BootVolumeType" = "{\"descriptorType\":\"enumstring\",\"values\":[\"SCSI\",\"IDE\",\"PARAVIRTUALIZED\"],\"defaultValue\":\"PARAVIRTUALIZED\",\"source\":\"GLOBAL\"}", +# } +# } + +# data "oci_core_compute_image_capability_schemas" "talos_amd64" { +# compartment_id = var.tenancy_ocid +# image_id = oci_core_image.talos_amd64.id +# } + +# data "oci_core_compute_global_image_capability_schemas_versions" "default" { +# compute_global_image_capability_schema_id = data.oci_core_compute_global_image_capability_schema.default.id +# } + +# data "oci_core_compute_global_image_capability_schemas" "default" { +# display_name = "OCI.ComputeGlobalImageCapabilitySchema" +# } + +# data "oci_core_compute_global_image_capability_schema" "default" { +# compute_global_image_capability_schema_id = data.oci_core_compute_global_image_capability_schemas.default.compute_global_image_capability_schemas[0].id +# } + +# data "oci_core_compute_global_image_capability_schemas_version" "default" { +# compute_global_image_capability_schema_id = data.oci_core_compute_global_image_capability_schema.default.id +# compute_global_image_capability_schema_version_name = data.oci_core_compute_global_image_capability_schemas_versions.default.compute_global_image_capability_schema_versions[0].name +# } + +# data "oci_core_compute_image_capability_schema" "test_compute_image_capability_schema" { +# compute_image_capability_schema_id = oci_core_compute_image_capability_schema.test_compute_image_capability_schema.id +# is_merge_enabled = "true" +# } + +# resource "oci_core_compute_image_capability_schema" "test_compute_image_capability_schema" { +# compartment_id = var.tenancy_ocid +# compute_global_image_capability_schema_version_name = data.oci_core_compute_global_image_capability_schemas_versions.test_compute_global_image_capability_schemas_versions_datasource.compute_global_image_capability_schema_versions[0].name +# display_name = "displayName" +# image_id = oci_core_image.talos_amd64.id + +# schema_data = { +# "Storage.BootVolumeType" = "{\"descriptorType\":\"enumstring\",\"values\":[\"SCSI\",\"IDE\",\"PARAVIRTUALIZED\"],\"defaultValue\":\"PARAVIRTUALIZED\",\"source\":\"GLOBAL\"}", +# } +# } + +# data "oci_core_compute_global_image_capability_schemas_version" "test_compute_global_image_capability_schemas_version_datasource" { +# compute_global_image_capability_schema_id = data.oci_core_compute_global_image_capability_schema.test_compute_global_image_capability_schema_datasource.id +# compute_global_image_capability_schema_version_name = data.oci_core_compute_global_image_capability_schemas_versions.test_compute_global_image_capability_schemas_versions_datasource.compute_global_image_capability_schema_versions[0].name +# } + +# data "oci_core_compute_global_image_capability_schemas_versions" "test_compute_global_image_capability_schemas_versions_datasource" { +# compute_global_image_capability_schema_id = data.oci_core_compute_global_image_capability_schema.test_compute_global_image_capability_schema_datasource.id +# } + +# data "oci_core_compute_global_image_capability_schema" "test_compute_global_image_capability_schema_datasource" { +# compute_global_image_capability_schema_id = data.oci_core_compute_global_image_capability_schemas.test_compute_global_image_capability_schemas_datasource.compute_global_image_capability_schemas[0].id +# } + +# data "oci_core_compute_global_image_capability_schemas" "test_compute_global_image_capability_schemas_datasource" { +# } diff --git a/oracle/prepare/network-secgroup.tf b/oracle/prepare/network-secgroup.tf index 01b165b..6b50dbf 100644 --- a/oracle/prepare/network-secgroup.tf +++ b/oracle/prepare/network-secgroup.tf @@ -4,48 +4,39 @@ resource "oci_core_default_security_list" "main" { manage_default_resource_id = oci_core_vcn.main.default_security_list_id display_name = "DefaultSecurityList" - dynamic "egress_security_rules" { - for_each = ["0.0.0.0/0", "::/0"] - content { - destination = egress_security_rules.value - protocol = 6 - stateless = true - } - } - dynamic "egress_security_rules" { - for_each = ["0.0.0.0/0", "::/0"] - content { - destination = egress_security_rules.value - protocol = 17 - stateless = true - } + egress_security_rules { + protocol = 1 + destination = oci_core_vcn.main.cidr_block + stateless = true } egress_security_rules { - destination = "0.0.0.0/0" - protocol = "1" + protocol = 58 + destination = oci_core_vcn.main.ipv6cidr_blocks[0] + stateless = true } - - dynamic "ingress_security_rules" { + dynamic "egress_security_rules" { for_each = ["0.0.0.0/0", "::/0"] content { - source = ingress_security_rules.value - protocol = 6 - stateless = true - } - } - dynamic "ingress_security_rules" { - for_each = ["0.0.0.0/0", "::/0"] - content { - source = ingress_security_rules.value - protocol = 17 - stateless = true + protocol = "all" + destination = egress_security_rules.value + stateless = false } } ingress_security_rules { protocol = 1 - source = "0.0.0.0/0" + source = oci_core_vcn.main.cidr_block stateless = true + } + ingress_security_rules { + protocol = 58 + source = oci_core_vcn.main.ipv6cidr_blocks[0] + stateless = true + } + ingress_security_rules { + protocol = 1 + source = "0.0.0.0/0" + stateless = false icmp_options { type = 3 code = 4 @@ -58,24 +49,54 @@ resource "oci_core_network_security_group" "cilium" { compartment_id = var.compartment_ocid vcn_id = oci_core_vcn.main.id } -resource "oci_core_network_security_group_security_rule" "cilium_vxvlan" { - network_security_group_id = oci_core_network_security_group.cilium.id +resource "oci_core_network_security_group_security_rule" "cilium_vxvlan_in" { + for_each = toset([oci_core_vcn.main.cidr_block, oci_core_vcn.main.ipv6cidr_blocks[0]]) - protocol = "17" - direction = "INGRESS" - source = var.vpc_main_cidr - stateless = true + network_security_group_id = oci_core_network_security_group.cilium.id + protocol = "17" + direction = "INGRESS" + source = each.value + stateless = true udp_options { + source_port_range { + min = 8472 + max = 8472 + } + destination_port_range { + min = 8472 + max = 8472 + } + } +} +resource "oci_core_network_security_group_security_rule" "cilium_vxvlan_out" { + for_each = toset([oci_core_vcn.main.cidr_block, oci_core_vcn.main.ipv6cidr_blocks[0]]) + + network_security_group_id = oci_core_network_security_group.cilium.id + protocol = "17" + direction = "EGRESS" + destination = each.value + stateless = true + + udp_options { + source_port_range { + min = 8472 + max = 8472 + } + destination_port_range { + min = 8472 + max = 8472 + } } } resource "oci_core_network_security_group_security_rule" "cilium_health" { - network_security_group_id = oci_core_network_security_group.cilium.id + for_each = toset([oci_core_vcn.main.cidr_block, oci_core_vcn.main.ipv6cidr_blocks[0]]) - protocol = "6" - direction = "INGRESS" - source = var.vpc_main_cidr - stateless = true + network_security_group_id = oci_core_network_security_group.cilium.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false tcp_options { destination_port_range { @@ -92,12 +113,13 @@ resource "oci_core_network_security_group" "talos" { } resource "oci_core_network_security_group_security_rule" "talos" { - network_security_group_id = oci_core_network_security_group.talos.id + for_each = toset([oci_core_vcn.main.cidr_block, oci_core_vcn.main.ipv6cidr_blocks[0]]) - protocol = "6" - direction = "INGRESS" - source = var.vpc_main_cidr - stateless = true + network_security_group_id = oci_core_network_security_group.talos.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false tcp_options { destination_port_range { @@ -106,19 +128,35 @@ resource "oci_core_network_security_group_security_rule" "talos" { } } } +resource "oci_core_network_security_group_security_rule" "talos_admin" { + for_each = toset(var.whitelist_admins) -resource "oci_core_network_security_group_security_rule" "admin_ssh" { network_security_group_id = oci_core_network_security_group.talos.id - - protocol = "6" - direction = "INGRESS" - source = var.vpc_main_cidr - stateless = true + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false tcp_options { destination_port_range { - min = 22 - max = 22 + min = 50000 + max = 50001 + } + } +} +resource "oci_core_network_security_group_security_rule" "ntp" { + for_each = toset(["0.0.0.0/0", "::/0"]) + + network_security_group_id = oci_core_network_security_group.talos.id + protocol = "17" + direction = "EGRESS" + destination = each.value + stateless = false + + udp_options { + destination_port_range { + min = 123 + max = 123 } } } @@ -130,20 +168,54 @@ resource "oci_core_network_security_group" "contolplane_lb" { } resource "oci_core_network_security_group_security_rule" "kubernetes" { - network_security_group_id = oci_core_network_security_group.contolplane_lb.id + for_each = toset([oci_core_vcn.main.cidr_block, oci_core_vcn.main.ipv6cidr_blocks[0]]) - protocol = "6" - direction = "INGRESS" - source = var.vpc_main_cidr - stateless = true + network_security_group_id = oci_core_network_security_group.contolplane_lb.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false tcp_options { destination_port_range { - min = 80 - max = 80 + min = 6443 + max = 6443 } } } +resource "oci_core_network_security_group_security_rule" "kubernetes_admin" { + for_each = toset(var.whitelist_admins) + + network_security_group_id = oci_core_network_security_group.contolplane_lb.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false + + tcp_options { + destination_port_range { + min = 6443 + max = 6443 + } + } +} +resource "oci_core_network_security_group_security_rule" "kubernetes_talos_admin" { + for_each = toset(var.whitelist_admins) + + network_security_group_id = oci_core_network_security_group.contolplane_lb.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false + + tcp_options { + destination_port_range { + min = 50000 + max = 50000 + } + } +} + resource "oci_core_network_security_group" "contolplane" { display_name = "${var.project}-contolplane" @@ -151,12 +223,29 @@ resource "oci_core_network_security_group" "contolplane" { vcn_id = oci_core_vcn.main.id } resource "oci_core_network_security_group_security_rule" "contolplane_kubernetes" { - network_security_group_id = oci_core_network_security_group.contolplane.id + for_each = toset([oci_core_vcn.main.cidr_block, oci_core_vcn.main.ipv6cidr_blocks[0]]) - protocol = "6" - direction = "INGRESS" - source = "0.0.0.0/0" - stateless = true + network_security_group_id = oci_core_network_security_group.contolplane.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false + + tcp_options { + destination_port_range { + min = 6443 + max = 6443 + } + } +} +resource "oci_core_network_security_group_security_rule" "contolplane_kubernetes_admin" { + for_each = toset(var.whitelist_admins) + + network_security_group_id = oci_core_network_security_group.contolplane.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false tcp_options { destination_port_range { @@ -166,12 +255,13 @@ resource "oci_core_network_security_group_security_rule" "contolplane_kubernetes } } resource "oci_core_network_security_group_security_rule" "contolplane_etcd" { - network_security_group_id = oci_core_network_security_group.contolplane.id + for_each = toset([oci_core_vcn.main.cidr_block]) - protocol = "6" - direction = "INGRESS" - source = var.vpc_main_cidr - stateless = true + network_security_group_id = oci_core_network_security_group.contolplane.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false tcp_options { destination_port_range { @@ -186,13 +276,46 @@ resource "oci_core_network_security_group" "web" { compartment_id = var.compartment_ocid vcn_id = oci_core_vcn.main.id } -resource "oci_core_network_security_group_security_rule" "web_http" { - network_security_group_id = oci_core_network_security_group.web.id +resource "oci_core_network_security_group_security_rule" "web_http_health_check" { + for_each = toset([oci_core_vcn.main.cidr_block]) - protocol = "6" - direction = "INGRESS" - source = "0.0.0.0/0" - stateless = true + network_security_group_id = oci_core_network_security_group.web.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false + + tcp_options { + destination_port_range { + min = 80 + max = 80 + } + } +} +resource "oci_core_network_security_group_security_rule" "web_http_admin" { + for_each = toset(var.whitelist_admins) + + network_security_group_id = oci_core_network_security_group.web.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false + + tcp_options { + destination_port_range { + min = 80 + max = 80 + } + } +} +resource "oci_core_network_security_group_security_rule" "web_http" { + for_each = toset(var.whitelist_web) + + network_security_group_id = oci_core_network_security_group.web.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false tcp_options { destination_port_range { @@ -202,12 +325,13 @@ resource "oci_core_network_security_group_security_rule" "web_http" { } } resource "oci_core_network_security_group_security_rule" "web_https" { - network_security_group_id = oci_core_network_security_group.web.id + for_each = toset(var.whitelist_web) - protocol = "6" - direction = "INGRESS" - source = "0.0.0.0/0" - stateless = true + network_security_group_id = oci_core_network_security_group.web.id + protocol = "6" + direction = "INGRESS" + source = each.value + stateless = false tcp_options { destination_port_range { diff --git a/oracle/templates/controlplane.yaml.tpl b/oracle/templates/controlplane.yaml.tpl new file mode 100644 index 0000000..911038d --- /dev/null +++ b/oracle/templates/controlplane.yaml.tpl @@ -0,0 +1,52 @@ +version: v1alpha1 +debug: false +persist: true +machine: + certSANs: + - ${lbv4} + - ${lbv4_local} + - ${apiDomain} + kubelet: + extraArgs: + rotate-server-certificates: true + nodeIP: + validSubnets: ${format("%#v",split(",",nodeSubnets))} + network: + hostname: "${name}" + install: + wipe: false + sysctls: + net.core.somaxconn: 65535 + net.core.netdev_max_backlog: 4096 + systemDiskEncryption: + state: + provider: luks2 + keys: + - nodeID: {} + slot: 0 + ephemeral: + provider: luks2 + keys: + - nodeID: {} + slot: 0 + time: + servers: + - 169.254.169.254 +cluster: + controlPlane: + endpoint: https://${lbv4_local}:6443 + network: + dnsDomain: ${domain} + podSubnets: ${format("%#v",split(",",podSubnets))} + serviceSubnets: ${format("%#v",split(",",serviceSubnets))} + apiServer: + certSANs: + - ${lbv4} + - ${lbv4_local} + - ${apiDomain} + controllerManager: + extraArgs: + node-cidr-mask-size-ipv4: 24 + node-cidr-mask-size-ipv6: 112 + scheduler: {} + etcd: {} diff --git a/oracle/templates/web.yaml.tpl b/oracle/templates/web.yaml.tpl new file mode 100644 index 0000000..d85932c --- /dev/null +++ b/oracle/templates/web.yaml.tpl @@ -0,0 +1,59 @@ +version: v1alpha1 +debug: false +persist: true +machine: + type: worker + token: ${tokenMachine} + ca: + crt: ${caMachine} + certSANs: [] + kubelet: + extraArgs: + cloud-provider: external + rotate-server-certificates: true + clusterDNS: + - 169.254.2.53 + - 10.200.16.10 + nodeIP: + validSubnets: ${format("%#v",split(",",nodeSubnets))} + network: + interfaces: + - interface: eth0 + addresses: + - ${lbv4_web}/32 + dhcp: true + dhcpOptions + ipv6: true + - interface: dummy0 + addresses: + - 169.254.2.53/32 + - fd00::169:254:2:53/128 + install: + wipe: true + sysctls: + net.core.somaxconn: 65535 + net.core.netdev_max_backlog: 4096 + net.ipv4.tcp_keepalive_time: 600 + net.ipv4.tcp_keepalive_intvl: 60 + fs.inotify.max_user_instances: 256 + systemDiskEncryption: + state: + provider: luks2 + keys: + - nodeID: {} + slot: 0 + time: + servers: + - 169.254.169.254 +cluster: + controlPlane: + endpoint: https://${lbv4}:6443 + clusterName: ${clusterName} + network: + dnsDomain: ${domain} + serviceSubnets: ${format("%#v",split(",",serviceSubnets))} + proxy: + disabled: true + token: ${token} + ca: + crt: ${ca} diff --git a/oracle/variables.tf b/oracle/variables.tf index fd36d02..a1df960 100644 --- a/oracle/variables.tf +++ b/oracle/variables.tf @@ -24,6 +24,7 @@ data "terraform_remote_state" "prepare" { locals { project = data.terraform_remote_state.prepare.outputs.project + zone = data.terraform_remote_state.prepare.outputs.zones[0] nsg_contolplane_lb = data.terraform_remote_state.prepare.outputs.nsg_contolplane_lb network_lb = data.terraform_remote_state.prepare.outputs.network_lb @@ -36,3 +37,48 @@ locals { network_public = data.terraform_remote_state.prepare.outputs.network_public network_private = data.terraform_remote_state.prepare.outputs.network_private } + +variable "kubernetes" { + type = map(string) + default = { + podSubnets = "10.32.0.0/12,fd40:10:32::/102" + serviceSubnets = "10.200.0.0/22,fd40:10:200::/112" + nodeSubnets = "192.168.0.0/16" + domain = "cluster.local" + apiDomain = "api.cluster.local" + clusterName = "talos-k8s-oracle" + tokenMachine = "" + caMachine = "" + token = "" + ca = "" + } + # sensitive = true +} + +variable "controlplane" { + description = "Property of controlplane" + type = map(any) + default = { + count = 0, + type = "VM.Standard.E4.Flex" + ocpus = 1 + memgb = 3 + } +} + +variable "instances" { + description = "Map of region's properties" + type = map(any) + default = { + "jNdv:eu-amsterdam-1-AD-1" = { + web_count = 0, + web_instance_shape = "VM.Standard.E4.Flex", + web_instance_ocpus = 1, + web_instance_memgb = 3, + worker_count = 0, + worker_instance_shape = "VM.Standard.E2.1.Micro", + worker_instance_ocpus = 1, + worker_instance_memgb = 1, + }, + } +}