Define a basic but working complete k3s-kubernetes setup

This commit is contained in:
2025-03-10 01:21:02 +01:00
parent 2d93766c09
commit 1eb2ce341a
11 changed files with 404 additions and 32 deletions

1
.gitignore vendored
View File

@@ -1,6 +1,7 @@
*
!/.gitignore
!/**/*.tf
!/**/*.tftpl
!/.terraform.lock.hcl
!/modules/
!/modules/**/*/

62
.terraform.lock.hcl generated
View File

@@ -1,6 +1,68 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/external" {
version = "2.3.4"
constraints = "2.3.4"
hashes = [
"h1:XWkRZOLKMjci9/JAtE8X8fWOt7A4u+9mgXSUjc4Wuyo=",
"zh:037fd82cd86227359bc010672cd174235e2d337601d4686f526d0f53c87447cb",
"zh:0ea1db63d6173d01f2fa8eb8989f0809a55135a0d8d424b08ba5dabad73095fa",
"zh:17a4d0a306566f2e45778fbac48744b6fd9c958aaa359e79f144c6358cb93af0",
"zh:298e5408ab17fd2e90d2cd6d406c6d02344fe610de5b7dae943a58b958e76691",
"zh:38ecfd29ee0785fd93164812dcbe0664ebbe5417473f3b2658087ca5a0286ecb",
"zh:59f6a6f31acf66f4ea3667a555a70eba5d406c6e6d93c2c641b81d63261eeace",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:ad0279dfd09d713db0c18469f585e58d04748ca72d9ada83883492e0dd13bd58",
"zh:c69f66fd21f5e2c8ecf7ca68d9091c40f19ad913aef21e3ce23836e91b8cbb5f",
"zh:d4a56f8c48aa86fc8e0c233d56850f5783f322d6336f3bf1916e293246b6b5d4",
"zh:f2b394ebd4af33f343835517e80fc876f79361f4688220833bc3c77655dd2202",
"zh:f31982f29f12834e5d21e010856eddd19d59cd8f449adf470655bfd19354377e",
]
}
provider "registry.terraform.io/hashicorp/random" {
version = "3.7.1"
constraints = "3.7.1"
hashes = [
"h1:/qtweZW2sk0kBNiQM02RvBXmlVdI9oYqRMCyBZ8XA98=",
"zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585",
"zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef",
"zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d",
"zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6",
"zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1",
"zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e",
"zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601",
"zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436",
"zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d",
"zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea",
]
}
provider "registry.terraform.io/hetznercloud/hcloud" {
version = "1.50.0"
constraints = "1.50.0"
hashes = [
"h1:z5J9wgkt9xIKlr699hWCjHSS7K4bYKWWnGCg2T/YNmg=",
"zh:0bd650fb52e272f74eda5053a7bb62f0fd92182f57ad3ef742abe165cb8cac98",
"zh:1c36667aa89b672a96c0df3d3c613e80916a2d0944b1a1f9112065f40630b689",
"zh:21f90683890ea7a184b0ac55efd52911694ba86c58898bc8bbe87ee2507bb1eb",
"zh:24349d483a6ff97420d847433553fa031f68f99b9ead4ebb3592fc8955ef521f",
"zh:3fffd83c450bea2b382a986501ae51a4d3e6530eda48ed9ca74d518e4a909c37",
"zh:43d7de1dc4c50fae99d6c4ab4bb394608948091f5b53ddb29bc65deead9dc8a6",
"zh:47a37d5fec79dd8bc9cab2c892bc59e135b86cb51eebe2b01cdb40afac7ed777",
"zh:6efeb9530b8f57618c43f0b294b983d06cce43e9423bdd737eed81db913edb80",
"zh:7511ace4b33baddfc452ef95a634d83b92bfbfaa23cb30403899e95b64727075",
"zh:7bade77104ed8788c9b5171c7daae6ab6c011b3c40b152274fda803bf0bf2707",
"zh:83bce3ff9a1bd52a340a6ebdd2e2b731ec6fb86811ef0ed8a8264daf9d7beb61",
"zh:a09d5fce4c8d33e10b9a19318c965076db2d8ed5f62f5feb3e7502416f66d7bf",
"zh:c942832b80270eb982eeb9cc14f30a437db5fd28faf37d6aa32ec2cd345537d6",
"zh:e2c1812f2e1f9fac17c7551d4ab0efb713b6d751087c18b84b8acd542f587459",
]
}
provider "registry.terraform.io/timohirt/hetznerdns" {
version = "2.2.0"
constraints = "2.2.0"

119
main.tf
View File

@@ -1,3 +1,54 @@
resource "hcloud_ssh_key" "this" {
for_each = var.ssh_keys
name = each.key
public_key = each.value
}
resource "hcloud_primary_ip" "k8s_ipv4" {
name = "k8s_primary_ipv4"
datacenter = "fsn1-dc14"
type = "ipv4"
assignee_type = "server"
auto_delete = false # change to true and apply before deleting!
}
resource "hcloud_primary_ip" "k8s_ipv6" {
name = "k8s_primary_ipv6"
datacenter = "fsn1-dc14"
type = "ipv6"
assignee_type = "server"
auto_delete = false # change to true and apply before deleting!
}
data "external" "my_ip" {
program = [
"sh",
"-c",
"(dig TXT +short -4 o-o.myaddr.l.google.com @ns1.google.com && dig TXT +short -6 o-o.myaddr.l.google.com @ns1.google.com) | jq '{(.): .}' | jq -s add"
]
}
module "k8s" {
source = "./modules/hetzner/kubernetes"
development_ips = [ for ip in data.external.my_ip.result : ip ]
name = "cluster1"
ssh_keys = [for o in hcloud_ssh_key.this : o.id]
# Only odd numbers of servers make any sense
servers = [{
ipv4_id = hcloud_primary_ip.k8s_ipv4.id
ipv6_id = hcloud_primary_ip.k8s_ipv6.id
type = "cax11"
location = "fsn1"
}]
agents = [{
type = "cax11"
location = "fsn1"
count = 1
}]
}
locals {
dns_zones = {
# costs-table:
@@ -12,58 +63,72 @@ locals {
"goperte.de" = {
zone_ttl = 900
records = [
{ name = "@", type = "A", value = "62.138.6.205" },
{ name = "*", type = "A", value = "62.138.6.205" },
{ name = "@", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "*", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "@", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
{ name = "*", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
]
},
"nehrke.info" = {
zone_ttl = 3600
zone_ttl = 900
records = [
{ name = "@", ttl = 900, type = "A", value = "62.138.6.205" },
{ name = "*", ttl = 900, type = "A", value = "62.138.6.205" },
{ name = "@", type = "MX", value = "1 smtp.google.com." },
{ name = "@", type = "TXT", value = "v=spf1 include:_spf.google.com a mx ~all" },
{ name = "_dmarc", type = "TXT", value = "v=DMARC1; p=none;" },
{ name = "google._domainkey", type = "TXT", value = var.nehrke_info_dkim },
{ name = "@", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "*", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "@", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
{ name = "*", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
{ name = "@", ttl = 86400, type = "MX", value = "1 smtp.google.com." },
{ name = "@", ttl = 86400, type = "TXT", value = "v=spf1 include:_spf.google.com a mx ~all" },
{ name = "_dmarc", ttl = 86400, type = "TXT", value = "v=DMARC1; p=none;" },
{ name = "google._domainkey", ttl = 86400, type = "TXT", value = var.nehrke_info_dkim },
]
},
"sozpaedil.net" = {
zone_ttl = 3600
zone_ttl = 900
records = [
{ name = "@", ttl = 900, type = "A", value = "62.138.6.205" },
{ name = "*", ttl = 900, type = "A", value = "62.138.6.205" },
{ name = "@", type = "MX", value = "1 smtp.google.com." },
{ name = "@", type = "TXT", value = "v=spf1 include:_spf.google.com a mx ~all" },
{ name = "_dmarc", type = "TXT", value = "v=DMARC1; p=none;" },
{ name = "google._domainkey", type = "TXT", value = var.sozpaedil_net_dkim },
{ name = "@", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "*", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "@", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
{ name = "*", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
{ name = "@", ttl = 86400, type = "MX", value = "1 smtp.google.com." },
{ name = "@", ttl = 86400, type = "TXT", value = "v=spf1 include:_spf.google.com a mx ~all" },
{ name = "_dmarc", ttl = 86400, type = "TXT", value = "v=DMARC1; p=none;" },
{ name = "google._domainkey", ttl = 86400, type = "TXT", value = var.sozpaedil_net_dkim },
]
},
"tovot.de" = {
zone_ttl = 900
records = [
{ name = "@", type = "A", value = "62.138.6.205" },
{ name = "*", type = "A", value = "62.138.6.205" },
{ name = "@", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "*", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "@", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
{ name = "*", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
]
},
"tovot.net" = {
zone_ttl = 900
records = [
{ name = "@", type = "A", value = "62.138.6.205" },
{ name = "*", type = "A", value = "62.138.6.205" },
{ name = "@", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "*", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "@", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
{ name = "*", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
]
},
"tovot.org" = {
zone_ttl = 900
records = [
{ name = "@", type = "A", value = "62.138.6.205" },
{ name = "*", type = "A", value = "62.138.6.205" },
{ name = "@", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "*", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "@", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
{ name = "*", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
]
},
"xn--alleingnger-r8a.de" = {
zone_ttl = 900
records = [
{ name = "@", type = "A", value = "62.138.6.205" },
{ name = "*", type = "A", value = "62.138.6.205" },
{ name = "@", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "*", type = "A", value = hcloud_primary_ip.k8s_ipv4.ip_address },
{ name = "@", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
{ name = "*", type = "AAAA", value = hcloud_primary_ip.k8s_ipv6.ip_address },
]
},
}
@@ -77,9 +142,3 @@ module "dns" {
records = lookup(each.value, "records")
}
resource "hcloud_ssh_key" "this" {
for_each = var.ssh_keys
name = each.key
public_key = each.value
}

View File

@@ -0,0 +1,25 @@
#cloud-config
packages:
- curl
users:
- name: cluster
shell: /bin/bash
runcmd:
# configure correct routing via NAT
- ip route add default via ${network_gateway}
- NIC=$(ifconfig | grep -q enp7s0 && echo enp7s0 || echo ens10)
- echo "[Match]" > /etc/systemd/network/10-$NIC.network
- echo "Name=$NIC" >> /etc/systemd/network/10-$NIC.network
- echo "[Network]" >> /etc/systemd/network/10-$NIC.network
- echo "DHCP=yes" >> /etc/systemd/network/10-$NIC.network
- echo "Gateway=${network_gateway}" >> /etc/systemd/network/10-$NIC.network
- sed -e "s/#DNS=/DNS=${dns_servers}/" -i /etc/systemd/resolved.conf
- systemctl restart systemd-resolved
# update system dependency-lists
- apt-get update -y
# install k3s but do not start yet
- curl -sfL https://get.k3s.io | tee install-k3s | INSTALL_K3S_SKIP_ENABLE=true sh -s -
# wait for the server node to be ready by trying to connect to it
- until curl -k https://${server_ip}:6443; do sleep 5; done
# Enable and start k3s-agent
- cat install-k3s | K3S_TOKEN=${k3s_token} sh -s - agent --server https://${server_ip}:6443

View File

@@ -0,0 +1,141 @@
locals {
network = "10.0.0.0/16"
subnet_eu_central = "10.0.0.0/24"
servers = {
for idx, config in var.servers : "${var.name}-server-${idx + 1}" => merge(
config,
{
ip = cidrhost(local.subnet_eu_central, idx + 2)
first_ip = idx == 0 ? "" : cidrhost(local.subnet_eu_central, 2)
}
)
}
agents = merge([
for idx, config in var.agents : {
for n in range(0, config.count) : "${var.name}-agent-${idx + 1}-${n + 1}" => merge(
config,
{ ip = cidrhost(local.subnet_eu_central, 255 - (idx * 20) - n - 1) }
)
}
]...)
}
resource "hcloud_network" "this" {
name = var.name
ip_range = local.network
}
resource "hcloud_network_subnet" "this" {
type = "cloud"
network_id = hcloud_network.this.id
network_zone = "eu-central"
ip_range = local.subnet_eu_central
}
resource "hcloud_network_route" "this" {
network_id = hcloud_network.this.id
destination = "0.0.0.0/0"
gateway = cidrhost(local.subnet_eu_central, 2)
}
resource "random_string" "k3s_token" {
length = 100
special = false
}
resource "hcloud_firewall" "this" {
name = var.name
rule {
direction = "in"
protocol = "icmp"
source_ips = ["0.0.0.0/0", "::/0"]
}
rule {
direction = "in"
protocol = "tcp"
port = "22"
source_ips = ["0.0.0.0/0", "::/0"]
}
rule {
direction = "in"
protocol = "tcp"
port = "80"
source_ips = ["0.0.0.0/0", "::/0"]
}
rule {
direction = "in"
protocol = "tcp"
port = "443"
source_ips = ["0.0.0.0/0", "::/0"]
}
rule {
direction = "in"
protocol = "tcp"
port = "6443"
source_ips = concat([local.network], var.development_ips)
}
dynamic "rule" {
for_each = length(var.development_ips) == 0 ? {} : { ips = 1 }
content {
direction = "in"
protocol = "tcp"
port = "1022"
source_ips = var.development_ips
}
}
}
resource "hcloud_server" "server" {
depends_on = [hcloud_network_subnet.this]
for_each = local.servers
name = each.key
image = "ubuntu-24.04"
server_type = each.value.type
location = each.value.location
ssh_keys = var.ssh_keys
public_net {
ipv4 = each.value.ipv4_id
ipv6 = each.value.ipv6_id
}
network {
network_id = hcloud_network.this.id
ip = each.value.ip
}
user_data = templatefile(
"${path.module}/server-init.yaml.tftpl",
{
network_ip_range = local.network
k3s_token = random_string.k3s_token.result
first_ip = each.value.first_ip
}
)
firewall_ids = [hcloud_firewall.this.id]
}
resource "hcloud_server" "agent" {
depends_on = [hcloud_server.server]
for_each = local.agents
name = each.key
image = "ubuntu-24.04"
server_type = each.value.type
location = each.value.location
ssh_keys = var.ssh_keys
public_net {
ipv4_enabled = false
ipv6_enabled = false
}
network {
network_id = hcloud_network.this.id
ip = each.value.ip
}
user_data = templatefile(
"${path.module}/agent-init.yaml.tftpl",
{
server_ip = cidrhost(local.subnet_eu_central, 2)
network_gateway = cidrhost(local.subnet_eu_central, 1)
dns_servers = "8.8.8.8 8.8.4.4"
k3s_token = random_string.k3s_token.result
}
)
}

View File

View File

@@ -0,0 +1,29 @@
#cloud-config
packages:
- curl
users:
- name: cluster
shell: /bin/bash
runcmd:
# update system dependency-lists
- apt-get update -y
# configure NAT
- echo '#!/bin/bash' > /etc/networkd-dispatcher/routable.d/10-eth0-post-up
- echo 'echo 1 > /proc/sys/net/ipv4/ip_forward' >> /etc/networkd-dispatcher/routable.d/10-eth0-post-up
- echo 'iptables -t nat -A POSTROUTING -s ${network_ip_range} -o eth0 -j MASQUERADE' >> /etc/networkd-dispatcher/routable.d/10-eth0-post-up
- chmod +x /etc/networkd-dispatcher/routable.d/10-eth0-post-up
- /etc/networkd-dispatcher/routable.d/10-eth0-post-up
# install k3s but do not start yet
- curl -sfL https://get.k3s.io | tee install-k3s | INSTALL_K3S_SKIP_ENABLE=true sh -s -
%{ if first_ip != "" ~}
- until curl -k https://${first_ip}:6443; do sleep 5; done
%{ endif ~}
# Enable and start k3s-server
%{ if first_ip == "" ~}
- cat install-k3s | K3S_TOKEN=${k3s_token} sh -s - server --cluster-init
%{~ else ~}
- cat install-k3s | INSTALL_K3S_SKIP_DOWNLOAD=true K3S_TOKEN=${k3s_token} sh -s - server --server https://${first_ip}:6443
%{~ endif }
- chown cluster:cluster /etc/rancher/k3s/k3s.yaml
- chown cluster:cluster /var/lib/rancher/k3s/server
- chown cluster:cluster /var/lib/rancher/k3s/server/node-token

View File

@@ -0,0 +1,29 @@
variable "name" {
type = string
}
variable "ssh_keys" {
type = list(string)
}
variable "servers" {
type = list(object({
ipv4_id = number
ipv6_id = number
type = string
location = string
}))
}
variable "agents" {
type = list(object({
count = optional(number, 1)
type = string
location = string
}))
}
variable "development_ips" {
type = list(string)
default = []
}

View File

@@ -0,0 +1,14 @@
terraform {
required_providers {
hcloud = {
source = "hetznercloud/hcloud"
version = "1.50.0"
}
random = {
source = "hashicorp/random"
version = "3.7.1"
}
}
}

View File

@@ -19,3 +19,4 @@ variable "sozpaedil_net_dkim" {
variable "ssh_keys" {
type = map(string)
}

View File

@@ -1,13 +1,21 @@
terraform {
required_providers {
hetznerdns = {
source = "timohirt/hetznerdns"
source = "timohirt/hetznerdns"
version = "2.2.0"
}
hcloud = {
source = "hetznercloud/hcloud"
source = "hetznercloud/hcloud"
version = "1.50.0"
}
random = {
source = "hashicorp/random"
version = "3.7.1"
}
external = {
source = "hashicorp/external"
version = "2.3.4"
}
}
}
@@ -18,3 +26,6 @@ provider "hetznerdns" {
provider "hcloud" {
token = var.hetzner_cloud_apitoken
}
provider "random" {}
provider "external" {}