Setup k3s with the help of ansible after the infra-provisioning

This change is huge! It allows me to manage the k3s installation on the
hetzner-cloud I setup via terraform.

.gitignore

@@ -1,7 +1,183 @@
-*
+inventory.*
-!/.gitignore
+!inventory.ini.tftpl
-!/**/*.tf
+README.*
-!/**/*.tftpl
+!README.adoc
-!/.terraform.lock.hcl
+
-!/modules/
+# Created by https://www.toptal.com/developers/gitignore/api/vim,ansible,jetbrains,terraform
-!/modules/**/*/
+# Edit at https://www.toptal.com/developers/gitignore?templates=vim,ansible,jetbrains,terraform
+
+### Ansible ###
+*.retry
+
+### JetBrains ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### JetBrains Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+# Azure Toolkit for IntelliJ plugin
+# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
+.idea/**/azureSettings.xml
+
+### Terraform ###
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+# End of https://www.toptal.com/developers/gitignore/api/vim,ansible,jetbrains,terraform
+

.terraform.lock.hcl

@@ -21,6 +21,25 @@ provider "registry.terraform.io/hashicorp/external" {
   ]
 }
 
+provider "registry.terraform.io/hashicorp/local" {
+  version = "2.5.3"
+  hashes = [
+    "h1:1Nkh16jQJMp0EuDmvP/96f5Unnir0z12WyDuoR6HjMo=",
+    "zh:284d4b5b572eacd456e605e94372f740f6de27b71b4e1fd49b63745d8ecd4927",
+    "zh:40d9dfc9c549e406b5aab73c023aa485633c1b6b730c933d7bcc2fa67fd1ae6e",
+    "zh:6243509bb208656eb9dc17d3c525c89acdd27f08def427a0dce22d5db90a4c8b",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:885d85869f927853b6fe330e235cd03c337ac3b933b0d9ae827ec32fa1fdcdbf",
+    "zh:bab66af51039bdfcccf85b25fe562cbba2f54f6b3812202f4873ade834ec201d",
+    "zh:c505ff1bf9442a889ac7dca3ac05a8ee6f852e0118dd9a61796a2f6ff4837f09",
+    "zh:d36c0b5770841ddb6eaf0499ba3de48e5d4fc99f4829b6ab66b0fab59b1aaf4f",
+    "zh:ddb6a407c7f3ec63efb4dad5f948b54f7f4434ee1a2607a49680d494b1776fe1",
+    "zh:e0dafdd4500bec23d3ff221e3a9b60621c5273e5df867bc59ef6b7e41f5c91f6",
+    "zh:ece8742fd2882a8fc9d6efd20e2590010d43db386b920b2a9c220cfecc18de47",
+    "zh:f4c6b3eb8f39105004cf720e202f04f57e3578441cfb76ca27611139bc116a82",
+  ]
+}
+
 provider "registry.terraform.io/hashicorp/random" {
   version     = "3.7.1"
   constraints = "3.7.1"

README.adoc

@@ -0,0 +1,80 @@
+= Hetzer Infra
+:icons: font
+
+This repository is meant to setup my kubernetes-cluster on Hetzner Cloud.
+The setup is split into 2 dedicated parts:
+
+. Create necessary machines in Hetzner Cloud via terraform.
+. Install/Upgrade kubernetes-cluster and other software via ansible.
+
+== Preparation
+
+. Ensure `terraform` is installed
+. Ensure `ansible` is installed
+. Create `config.auto.tfvars` with all the needed configuration-secrets (Stored in password-manager)
+
+== Setup
+
+In most cases it should be save to follow this guide:
+
+[source,bash]
+----
+ansible-galaxy install -r requirements.yml # <1>
+terraform apply # <2>
+ansible-playbook k3s.orchestration.site -i inventory.ini # <3>
+ansible-playbook download-kube-config.yml -i inventory.ini # <4>
+ansible-playbook k3s.orchestration.upgrade -i inventory.ini # <5>
+----
+
+<1> Install required ansible collections to create a k3s-cluster (can be omitted in subsequent runs)
+<2> Setup infrastructure and create/update inventory.ini (This might take some time, even after it's "ready")
+<3> Install k3s
+<4> Download the kube-config to .kube/config
+<5> Update k3s when necessary
+
+[IMPORTANT]
+Step 4 will override any existing kube config this might destroy any existing settings!
+
+== Enlarge / Reduce size of cluster
+
+Increase::
+--
+. Simply adjust the number of agents/servers in your `config.auto.tfvars`.
+. Run steps 2 & 3 of the setup again
+--
+Decrease::
+--
+If you want shrink the cluster **DO NOT** reduce the agent-amount directly!
+Instead proceed as the following:
+
+. Open k9s and go to `:nodes`
+. Select the highest agent and press `r` to drain it
+. Afterward that succeeded delete it with `Ctrl-d`
+. Finally reduce the amount of agents in terraform and apply the change
+--
+
+== Responsibilities
+
+The terraform scripts are responsible for:
+
+* Creation of network for the kubernetes-cluster
+** A public subnet exposed to the internet for the kubernetes-servers
+** A private subnet for the kubernetes-agents
+* Routing between the networks
+* Firewall rules to block everything from the servers except of:
+** ping (protocol: `icmp`)
+** kubernetes api (Usually port `6443`)
+** ssh (I prefer to use a non-standard port since I want to provide a git-server on port `22`)
+** public services, e.g. http and https (port `80` and `443`)
+* Creating the kubernetes-servers in the public subnet
+* Creating the kubernetes-agents in the private subnet
+* Setting up routing on all servers
+* Setup SSH-connections
+* Creating DNS-records in Hetzer Cloud
+
+The ansible scripts are responsible for:
+
+* Installing k3s
+* Keep the software up-to-date
+
+

download-kube-config.yml

@@ -0,0 +1,15 @@
+- hosts: server[0]
+  tasks:
+  - name: Download kube-config
+    fetch:
+      src: /etc/rancher/k3s/k3s.yaml
+      dest: "{{ lookup('env', 'HOME') }}/.kube/config"
+      flat: true
+- hosts: localhost
+  tasks:
+  - name: Use correct ip-address for k8s-cluster
+    lineinfile:
+      path: "{{ lookup('env', 'HOME') }}/.kube/config"
+      regexp: '^(\s*server: https://).*(:\d+)$'
+      line: \g<1>{{ hostvars[groups['server'][0]]['api_endpoint'] }}\g<2>
+      backrefs: yes

inventory.ini.tftpl

@@ -0,0 +1,29 @@
+[server]
+%{for ip in server_ips~}
+${ip}
+%{endfor~}
+
+[server:vars]
+ansible_user=root
+ansible_ssh_common_args='-o StrictHostKeyChecking=accept-new'
+%{if length(server_ips) > 0~}
+api_endpoint=${server_ips[0]}
+%{endif~}
+k3s_version=${k3s_version}
+
+[agent]
+%{for ip in agent_ips~}
+${ip}
+%{endfor~}
+
+[agent:vars]
+ansible_user=root
+%{if length(server_ips) > 0~}
+ansible_ssh_common_args='-o StrictHostKeyChecking=accept-new -o ProxyCommand="ssh -p 22 -W %h:%p -q root@${server_ips[0]}"'
+api_endpoint=${server_ips[0]}
+%{endif~}
+k3s_version=${k3s_version}
+
+[k3s_cluster:children]
+server
+agent

main.tf

@@ -55,6 +55,15 @@ module "k8s" {
   }
 }
 
+resource "local_file" "ansible_inventory" {
+  filename = "${path.module}/inventory.ini"
+  content = templatefile("./inventory.ini.tftpl", {
+    server_ips  = module.k8s.server_ips_v4,
+    agent_ips   = module.k8s.agent_ips_v4,
+    k3s_version = var.k3s_version,
+  })
+}
+
 locals {
   dns_zones = {
     # costs-table (incl. taxes):

modules/hetzner/kubernetes/agent-init.yaml.tftpl

@@ -15,11 +15,3 @@ runcmd:
 - echo "Gateway=${network_gateway}" >> /etc/systemd/network/10-$NIC.network
 - sed -e "s/#DNS=/DNS=${dns_servers}/" -i /etc/systemd/resolved.conf
 - systemctl restart systemd-resolved
-# update system dependency-lists
-- apt-get update -y
-# install k3s but do not start yet
-- curl -sfL https://get.k3s.io | tee install-k3s | INSTALL_K3S_SKIP_ENABLE=true sh -s -
-# wait for the server node to be ready by trying to connect to it
-- until curl -k https://${server_ip}:6443; do sleep 5; done
-# Enable and start k3s-agent
-- cat install-k3s | K3S_TOKEN=${k3s_token} sh -s - agent --server https://${server_ip}:6443

modules/hetzner/kubernetes/main.tf

@@ -68,10 +68,14 @@ resource "hcloud_firewall" "this" {
     }
   }
 }
-
 resource "hcloud_server" "server" {
   depends_on  = [hcloud_network_subnet.this]
   for_each    = local.servers
+
+  lifecycle {
+    ignore_changes = [ user_data ]
+  }
+
   name        = each.key
   image       = "ubuntu-24.04"
   server_type = each.value.type
@@ -100,6 +104,10 @@ resource "hcloud_server" "agent" {
   depends_on = [hcloud_server.server]
   for_each   = local.agents
 
+  lifecycle {
+    ignore_changes = [ user_data ]
+  }
+
   name        = each.key
   image       = "ubuntu-24.04"
   server_type = each.value.type

modules/hetzner/kubernetes/outputs.tf

@@ -0,0 +1,11 @@
+output "server_ips_v4" {
+  value = [for key, value in hcloud_server.server : value.ipv4_address]
+}
+
+output "server_ips_v6" {
+  value = [for key, value in hcloud_server.server : value.ipv6_address]
+}
+
+output "agent_ips_v4" {
+  value = flatten([for key, value in hcloud_server.agent : value.network.*.ip])
+}

modules/hetzner/kubernetes/server-init.yaml.tftpl

@@ -5,25 +5,9 @@ users:
 - name: cluster
   shell: /bin/bash
 runcmd:
-# update system dependency-lists
-- apt-get update -y
 # configure NAT
 - echo '#!/bin/bash' > /etc/networkd-dispatcher/routable.d/10-eth0-post-up
 - echo 'echo 1 > /proc/sys/net/ipv4/ip_forward' >> /etc/networkd-dispatcher/routable.d/10-eth0-post-up
 - echo 'iptables -t nat -A POSTROUTING -s ${network_ip_range} -o eth0 -j MASQUERADE' >> /etc/networkd-dispatcher/routable.d/10-eth0-post-up
 - chmod +x /etc/networkd-dispatcher/routable.d/10-eth0-post-up
 - /etc/networkd-dispatcher/routable.d/10-eth0-post-up
-# install k3s but do not start yet
-- curl -sfL https://get.k3s.io | tee install-k3s | INSTALL_K3S_SKIP_ENABLE=true sh -s -
-%{ if first_ip != "" ~}
-- until curl -k https://${first_ip}:6443; do sleep 5; done
-%{ endif ~}
-# Enable and start k3s-server
-%{ if first_ip == "" ~}
-- cat install-k3s | K3S_TOKEN=${k3s_token} sh -s - server --cluster-init
-%{~ else ~}
-- cat install-k3s | INSTALL_K3S_SKIP_DOWNLOAD=true K3S_TOKEN=${k3s_token} sh -s - server --server https://${first_ip}:6443
-%{~ endif }
-- chown cluster:cluster /etc/rancher/k3s/k3s.yaml
-- chown cluster:cluster /var/lib/rancher/k3s/server
-- chown cluster:cluster /var/lib/rancher/k3s/server/node-token

requirements.yml

@@ -0,0 +1,4 @@
+collections:
+- name: https://github.com/k3s-io/k3s-ansible.git
+  type: git
+  version: 1.0.1

variables.tf

@@ -72,3 +72,8 @@ variable "k8s_agent_type" {
   type    = string
   default = "cax11"
 }
+
+variable "k3s_version" {
+  type        = string
+  description = "The k3s version to use."
+}