Move configuration of servers completely to ansible

With this change we no longer use user-data scripts on the provided
machines. That makes it way easier for me to handle all the
configuration, since I only have to run ansible. Furthermore this the
burdon to think what may went wrong, since ansible is easier to debug
than some arbitrary scripts which run at provisioning-time on the
machines.

With this change I should also think about restructuring the code a bit
as well. Since it's actually easier to provide the initial
software-stack for the cluster via ansible than via terraform, at least
as far as I can tell right now.

README.adoc

@@ -14,7 +14,7 @@ The setup is split into 2 dedicated parts:
 cd infra
 terraform init
 terraform apply
-sleep 300 # Wait 5 minutes to allow the user-data script to finish
+sleep 300 # Wait 5 minutes since the machines start _slow_ sometimes
 cd ../k3s
 ansible-galaxy install -r requirements.yml 
 ansible-playbook site.yml

infra/inventory.ini.tftpl

@@ -11,3 +11,7 @@ ${ip}
 [k3s_cluster:children]
 server
 agent
+
+[k3s_cluster:vars]
+network_cidr=${network_cidr}
+private_nat=${private_nat}

infra/main.tf

@@ -59,8 +59,10 @@ module "k8s" {
 resource "local_file" "ansible_inventory" {
   filename = "${path.module}/../k3s/inventory.ini"
   content = templatefile("./inventory.ini.tftpl", {
-    server_ips  = module.k8s.server_ips_v4,
-    agent_ips   = module.k8s.agent_ips_v4,
+    server_ips   = module.k8s.server_ips_v4,
+    agent_ips    = module.k8s.agent_ips_v4,
+    network_cidr = module.k8s.private_network_cidr,
+    private_nat  = module.k8s.private_network_nat,
   })
 }
 

infra/modules/hetzner/kubernetes/agent-init.yaml.tftpl

@@ -1,17 +0,0 @@
-#cloud-config
-packages:
-- curl
-users:
-- name: cluster
-  shell: /bin/bash
-runcmd:
-# configure correct routing via NAT
-- ip route add default via ${network_gateway}
-- NIC=$(ifconfig | grep -q enp7s0 && echo enp7s0 || echo ens10)
-- echo "[Match]" > /etc/systemd/network/10-$NIC.network
-- echo "Name=$NIC" >> /etc/systemd/network/10-$NIC.network
-- echo "[Network]" >> /etc/systemd/network/10-$NIC.network
-- echo "DHCP=yes" >> /etc/systemd/network/10-$NIC.network
-- echo "Gateway=${network_gateway}" >> /etc/systemd/network/10-$NIC.network
-- sed -e "s/#DNS=/DNS=${dns_servers}/" -i /etc/systemd/resolved.conf
-- systemctl restart systemd-resolved

infra/modules/hetzner/kubernetes/main.tf

@@ -89,14 +89,6 @@ resource "hcloud_server" "server" {
     network_id = hcloud_network.this.id
     ip         = each.value.ip
   }
-  user_data = templatefile(
-    "${path.module}/server-init.yaml.tftpl",
-    {
-      network_ip_range = local.network
-      k3s_token        = random_string.k3s_token.result
-      first_ip         = each.value.first_ip
-    }
-  )
   firewall_ids = [for firewall in hcloud_firewall.this : firewall.id]
 }
 
@@ -121,13 +113,4 @@ resource "hcloud_server" "agent" {
     network_id = hcloud_network.this.id
     ip         = each.value.ip
   }
-  user_data = templatefile(
-    "${path.module}/agent-init.yaml.tftpl",
-    {
-      server_ip       = cidrhost(local.subnet_eu_central, 2)
-      network_gateway = cidrhost(local.subnet_eu_central, 1)
-      dns_servers     = "8.8.8.8 8.8.4.4"
-      k3s_token       = random_string.k3s_token.result
-    }
-  )
 }

infra/modules/hetzner/kubernetes/outputs.tf

@@ -9,3 +9,11 @@ output "server_ips_v6" {
 output "agent_ips_v4" {
   value = flatten([for key, value in hcloud_server.agent : value.network.*.ip])
 }
+
+output "private_network_cidr" {
+  value = local.network
+}
+
+output "private_network_nat" {
+  value = cidrhost(local.subnet_eu_central, 1)
+}

infra/modules/hetzner/kubernetes/server-init.yaml.tftpl

@@ -1,17 +0,0 @@
-#cloud-config
-packages:
-- curl
-users:
-- name: cluster
-  shell: /bin/bash
-runcmd:
-# setup ssh over port 1022 instead of 22
-- sed -i /etc/ssh/sshd_config -e 's/^#\{,1\}\( *Port\) [0-9]\+$/\1 1022/'
-- systemctl daemon-reload
-- systemctl restart ssh.socket
-# configure NAT
-- echo '#!/bin/bash' > /etc/networkd-dispatcher/routable.d/10-eth0-post-up
-- echo 'echo 1 > /proc/sys/net/ipv4/ip_forward' >> /etc/networkd-dispatcher/routable.d/10-eth0-post-up
-- echo 'iptables -t nat -A POSTROUTING -s ${network_ip_range} -o eth0 -j MASQUERADE' >> /etc/networkd-dispatcher/routable.d/10-eth0-post-up
-- chmod +x /etc/networkd-dispatcher/routable.d/10-eth0-post-up
-- /etc/networkd-dispatcher/routable.d/10-eth0-post-up

k3s/group_vars/k3s_cluster/config.yml

@@ -1,3 +1,5 @@
 ansible_user: root
 api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}"
 
+# note the space between the IPs!
+dns_servers: 8.8.8.8 8.8.4.4

k3s/main.yml

@@ -1,3 +1,25 @@
+- hosts: server
+  gather_facts: no
+  roles:
+  - role: server-setup
+    tags:
+    - init
+    - add-server
+- hosts: agent
+  roles:
+  - role: agent-setup
+    tags:
+    - init
+    - add-agent
 - import_playbook: k3s.orchestration.site
+  tags:
+  - init
+  - add-server
+  - add-agent
 - import_playbook: download-kube-config.yml
+  tags:
+  - init
+  - config
 - import_playbook: k3s.orchestration.upgrade
+  tags:
+  - update

k3s/roles/agent-setup/defaults/main.yml

@@ -0,0 +1,3 @@
+dns_servers: 8.8.8.8 8.8.4.4
+network_config_path: /etc/systemd/network
+

k3s/roles/agent-setup/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: Restart resolved
+  service:
+    name: systemd-resolved
+    state: restarted

k3s/roles/agent-setup/tasks/main.yml

@@ -0,0 +1,29 @@
+- name: Set default network route
+  shell: "ip route add default via {{ private_nat }}"
+  ignore_errors: "yes"
+  when: ansible_facts['default_ipv4']['alias'] is not defined
+
+- name: Regather facts
+  setup:
+  when: ansible_facts['default_ipv4']['alias'] is not defined
+
+- name: Gather fact target_nic
+  set_fact:
+    target_nic: "{{ ansible_facts['default_ipv4']['alias'] }}"
+
+- name: Ensure path to configure default route
+  file:
+    path: "{{ network_config_path }}"
+    state: directory
+
+- name: Configure default route
+  template:
+    src: nic.network.j2
+    dest: "{{ network_config_path }}/10-{{target_nic}}.network"
+
+- name: Configure DNS servers
+  lineinfile:
+    path: /etc/systemd/resolved.conf
+    regexp: '^#?\s*DNS\s*=.*'
+    line: "DNS={{ dns_servers }}"
+  notify: "Restart resolved"

k3s/roles/agent-setup/templates/nic.network.j2

@@ -0,0 +1,5 @@
+[Match]
+Name={{ target_nic }}
+[Network]
+DHCP=yes
+Gateway={{ private_nat }}

k3s/roles/server-setup/defaults/main.yml

@@ -0,0 +1,2 @@
+ip_masquerade_path: /etc/networkd-dispatcher/routable.d
+ip_masquerade_script: 10-eth0-post-up

k3s/roles/server-setup/handlers/main.yml

@@ -0,0 +1,7 @@
+- name: Restart sshd
+  service:
+    name: ssh
+    state: restarted
+
+- name: Start ip-masquerade
+  script: "{{ ip_masquerade_path }}/{{ ip_masquerade_script }}"

k3s/roles/server-setup/tasks/main.yml

@@ -0,0 +1,50 @@
+- name: Set facts for target SSH-connection
+  set_fact:
+    target_ansible_port: "{{ ansible_port }}"
+
+- name: Check if SSH-connection is already adjusted
+  ping:
+  ignore_errors: "yes"
+  ignore_unreachable: "yes"
+  register: target_ssh
+
+- name: Set ansible_port to 22 when SSH-connection is not adjusted
+  set_fact:
+    ansible_port: "22"
+  when: target_ssh.unreachable is defined and
+        target_ssh.unreachable == True
+
+- name: Check if initial SSH-connection is active
+  ping:
+  when: target_ssh.unreachable is defined and
+        target_ssh.unreachable == True
+
+- name: Set SSH-port to 1022
+  lineinfile:
+    path: /etc/ssh/sshd_config
+    regexp: '^#?\s*Port\s+[0-9]+$'
+    line: Port 1022
+  notify: "Restart sshd"
+  when: target_ssh.unreachable is defined and
+        target_ssh.unreachable == True
+
+- name: Reset ansible_port to configured value
+  set_fact:
+    ansible_port: "{{ target_ansible_port }}"
+  when: target_ssh.unreachable is defined and
+        target_ssh.unreachable == True
+
+- name: Run deferred setup to gather facts
+  setup:
+
+- name: Ensure routable.d path to masquerade ips
+  file:
+    path: "{{ ip_masquerade_path }}"
+    state: directory
+
+- name: Configure NAT to masquerade ips
+  template:
+    src: ip-masquerade.sh.j2
+    dest: "{{ ip_masquerade_path }}/{{ ip_masquerade_script }}"
+    mode: u=rwx,g=rx,o=rx
+  notify: "Start ip-masquerade"

k3s/roles/server-setup/templates/ip-masquerade.sh.j2

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo 1 > /proc/sys/net/ipv4/ip_forward
+iptables -t nat -A POSTROUTING -s {{network_cidr}} -o eth0 -j MASQUERADE
+