diff --git a/README.adoc b/README.adoc index bbc03a1..fb5720f 100644 --- a/README.adoc +++ b/README.adoc @@ -14,7 +14,7 @@ The setup is split into 2 dedicated parts: cd infra terraform init terraform apply -sleep 300 # Wait 5 minutes to allow the user-data script to finish +sleep 300 # Wait 5 minutes since the machines start _slow_ sometimes cd ../k3s ansible-galaxy install -r requirements.yml ansible-playbook site.yml diff --git a/infra/inventory.ini.tftpl b/infra/inventory.ini.tftpl index 1b6cce7..642f0d2 100644 --- a/infra/inventory.ini.tftpl +++ b/infra/inventory.ini.tftpl @@ -11,3 +11,7 @@ ${ip} [k3s_cluster:children] server agent + +[k3s_cluster:vars] +network_cidr=${network_cidr} +private_nat=${private_nat} diff --git a/infra/main.tf b/infra/main.tf index f71c929..b6a3078 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -59,8 +59,10 @@ module "k8s" { resource "local_file" "ansible_inventory" { filename = "${path.module}/../k3s/inventory.ini" content = templatefile("./inventory.ini.tftpl", { - server_ips = module.k8s.server_ips_v4, - agent_ips = module.k8s.agent_ips_v4, + server_ips = module.k8s.server_ips_v4, + agent_ips = module.k8s.agent_ips_v4, + network_cidr = module.k8s.private_network_cidr, + private_nat = module.k8s.private_network_nat, }) } diff --git a/infra/modules/hetzner/kubernetes/agent-init.yaml.tftpl b/infra/modules/hetzner/kubernetes/agent-init.yaml.tftpl deleted file mode 100644 index 71a2ffd..0000000 --- a/infra/modules/hetzner/kubernetes/agent-init.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ -#cloud-config -packages: -- curl -users: -- name: cluster - shell: /bin/bash -runcmd: -# configure correct routing via NAT -- ip route add default via ${network_gateway} -- NIC=$(ifconfig | grep -q enp7s0 && echo enp7s0 || echo ens10) -- echo "[Match]" > /etc/systemd/network/10-$NIC.network -- echo "Name=$NIC" >> /etc/systemd/network/10-$NIC.network -- echo "[Network]" >> /etc/systemd/network/10-$NIC.network -- echo "DHCP=yes" >> /etc/systemd/network/10-$NIC.network -- echo "Gateway=${network_gateway}" >> /etc/systemd/network/10-$NIC.network -- sed -e "s/#DNS=/DNS=${dns_servers}/" -i /etc/systemd/resolved.conf -- systemctl restart systemd-resolved diff --git a/infra/modules/hetzner/kubernetes/main.tf b/infra/modules/hetzner/kubernetes/main.tf index 3524ea2..e36f5de 100644 --- a/infra/modules/hetzner/kubernetes/main.tf +++ b/infra/modules/hetzner/kubernetes/main.tf @@ -89,14 +89,6 @@ resource "hcloud_server" "server" { network_id = hcloud_network.this.id ip = each.value.ip } - user_data = templatefile( - "${path.module}/server-init.yaml.tftpl", - { - network_ip_range = local.network - k3s_token = random_string.k3s_token.result - first_ip = each.value.first_ip - } - ) firewall_ids = [for firewall in hcloud_firewall.this : firewall.id] } @@ -121,13 +113,4 @@ resource "hcloud_server" "agent" { network_id = hcloud_network.this.id ip = each.value.ip } - user_data = templatefile( - "${path.module}/agent-init.yaml.tftpl", - { - server_ip = cidrhost(local.subnet_eu_central, 2) - network_gateway = cidrhost(local.subnet_eu_central, 1) - dns_servers = "8.8.8.8 8.8.4.4" - k3s_token = random_string.k3s_token.result - } - ) } diff --git a/infra/modules/hetzner/kubernetes/outputs.tf b/infra/modules/hetzner/kubernetes/outputs.tf index d550f5c..aff48d3 100644 --- a/infra/modules/hetzner/kubernetes/outputs.tf +++ b/infra/modules/hetzner/kubernetes/outputs.tf @@ -9,3 +9,11 @@ output "server_ips_v6" { output "agent_ips_v4" { value = flatten([for key, value in hcloud_server.agent : value.network.*.ip]) } + +output "private_network_cidr" { + value = local.network +} + +output "private_network_nat" { + value = cidrhost(local.subnet_eu_central, 1) +} diff --git a/infra/modules/hetzner/kubernetes/server-init.yaml.tftpl b/infra/modules/hetzner/kubernetes/server-init.yaml.tftpl deleted file mode 100644 index 782d4f6..0000000 --- a/infra/modules/hetzner/kubernetes/server-init.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ -#cloud-config -packages: -- curl -users: -- name: cluster - shell: /bin/bash -runcmd: -# setup ssh over port 1022 instead of 22 -- sed -i /etc/ssh/sshd_config -e 's/^#\{,1\}\( *Port\) [0-9]\+$/\1 1022/' -- systemctl daemon-reload -- systemctl restart ssh.socket -# configure NAT -- echo '#!/bin/bash' > /etc/networkd-dispatcher/routable.d/10-eth0-post-up -- echo 'echo 1 > /proc/sys/net/ipv4/ip_forward' >> /etc/networkd-dispatcher/routable.d/10-eth0-post-up -- echo 'iptables -t nat -A POSTROUTING -s ${network_ip_range} -o eth0 -j MASQUERADE' >> /etc/networkd-dispatcher/routable.d/10-eth0-post-up -- chmod +x /etc/networkd-dispatcher/routable.d/10-eth0-post-up -- /etc/networkd-dispatcher/routable.d/10-eth0-post-up diff --git a/k3s/group_vars/k3s_cluster/config.yml b/k3s/group_vars/k3s_cluster/config.yml index db4f021..57bf432 100644 --- a/k3s/group_vars/k3s_cluster/config.yml +++ b/k3s/group_vars/k3s_cluster/config.yml @@ -1,3 +1,5 @@ ansible_user: root api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" +# note the space between the IPs! +dns_servers: 8.8.8.8 8.8.4.4 diff --git a/k3s/main.yml b/k3s/main.yml index 083368c..3ce4968 100644 --- a/k3s/main.yml +++ b/k3s/main.yml @@ -1,3 +1,25 @@ +- hosts: server + gather_facts: no + roles: + - role: server-setup + tags: + - init + - add-server +- hosts: agent + roles: + - role: agent-setup + tags: + - init + - add-agent - import_playbook: k3s.orchestration.site + tags: + - init + - add-server + - add-agent - import_playbook: download-kube-config.yml + tags: + - init + - config - import_playbook: k3s.orchestration.upgrade + tags: + - update diff --git a/k3s/roles/agent-setup/defaults/main.yml b/k3s/roles/agent-setup/defaults/main.yml new file mode 100644 index 0000000..4a03b27 --- /dev/null +++ b/k3s/roles/agent-setup/defaults/main.yml @@ -0,0 +1,3 @@ +dns_servers: 8.8.8.8 8.8.4.4 +network_config_path: /etc/systemd/network + diff --git a/k3s/roles/agent-setup/handlers/main.yml b/k3s/roles/agent-setup/handlers/main.yml new file mode 100644 index 0000000..92c6514 --- /dev/null +++ b/k3s/roles/agent-setup/handlers/main.yml @@ -0,0 +1,4 @@ +- name: Restart resolved + service: + name: systemd-resolved + state: restarted diff --git a/k3s/roles/agent-setup/tasks/main.yml b/k3s/roles/agent-setup/tasks/main.yml new file mode 100644 index 0000000..c07d6b0 --- /dev/null +++ b/k3s/roles/agent-setup/tasks/main.yml @@ -0,0 +1,29 @@ +- name: Set default network route + shell: "ip route add default via {{ private_nat }}" + ignore_errors: "yes" + when: ansible_facts['default_ipv4']['alias'] is not defined + +- name: Regather facts + setup: + when: ansible_facts['default_ipv4']['alias'] is not defined + +- name: Gather fact target_nic + set_fact: + target_nic: "{{ ansible_facts['default_ipv4']['alias'] }}" + +- name: Ensure path to configure default route + file: + path: "{{ network_config_path }}" + state: directory + +- name: Configure default route + template: + src: nic.network.j2 + dest: "{{ network_config_path }}/10-{{target_nic}}.network" + +- name: Configure DNS servers + lineinfile: + path: /etc/systemd/resolved.conf + regexp: '^#?\s*DNS\s*=.*' + line: "DNS={{ dns_servers }}" + notify: "Restart resolved" diff --git a/k3s/roles/agent-setup/templates/nic.network.j2 b/k3s/roles/agent-setup/templates/nic.network.j2 new file mode 100644 index 0000000..ec2570e --- /dev/null +++ b/k3s/roles/agent-setup/templates/nic.network.j2 @@ -0,0 +1,5 @@ +[Match] +Name={{ target_nic }} +[Network] +DHCP=yes +Gateway={{ private_nat }} diff --git a/k3s/roles/server-setup/defaults/main.yml b/k3s/roles/server-setup/defaults/main.yml new file mode 100644 index 0000000..d83d12b --- /dev/null +++ b/k3s/roles/server-setup/defaults/main.yml @@ -0,0 +1,2 @@ +ip_masquerade_path: /etc/networkd-dispatcher/routable.d +ip_masquerade_script: 10-eth0-post-up diff --git a/k3s/roles/server-setup/handlers/main.yml b/k3s/roles/server-setup/handlers/main.yml new file mode 100644 index 0000000..ef63412 --- /dev/null +++ b/k3s/roles/server-setup/handlers/main.yml @@ -0,0 +1,7 @@ +- name: Restart sshd + service: + name: ssh + state: restarted + +- name: Start ip-masquerade + script: "{{ ip_masquerade_path }}/{{ ip_masquerade_script }}" diff --git a/k3s/roles/server-setup/tasks/main.yml b/k3s/roles/server-setup/tasks/main.yml new file mode 100644 index 0000000..7d3e61c --- /dev/null +++ b/k3s/roles/server-setup/tasks/main.yml @@ -0,0 +1,50 @@ +- name: Set facts for target SSH-connection + set_fact: + target_ansible_port: "{{ ansible_port }}" + +- name: Check if SSH-connection is already adjusted + ping: + ignore_errors: "yes" + ignore_unreachable: "yes" + register: target_ssh + +- name: Set ansible_port to 22 when SSH-connection is not adjusted + set_fact: + ansible_port: "22" + when: target_ssh.unreachable is defined and + target_ssh.unreachable == True + +- name: Check if initial SSH-connection is active + ping: + when: target_ssh.unreachable is defined and + target_ssh.unreachable == True + +- name: Set SSH-port to 1022 + lineinfile: + path: /etc/ssh/sshd_config + regexp: '^#?\s*Port\s+[0-9]+$' + line: Port 1022 + notify: "Restart sshd" + when: target_ssh.unreachable is defined and + target_ssh.unreachable == True + +- name: Reset ansible_port to configured value + set_fact: + ansible_port: "{{ target_ansible_port }}" + when: target_ssh.unreachable is defined and + target_ssh.unreachable == True + +- name: Run deferred setup to gather facts + setup: + +- name: Ensure routable.d path to masquerade ips + file: + path: "{{ ip_masquerade_path }}" + state: directory + +- name: Configure NAT to masquerade ips + template: + src: ip-masquerade.sh.j2 + dest: "{{ ip_masquerade_path }}/{{ ip_masquerade_script }}" + mode: u=rwx,g=rx,o=rx + notify: "Start ip-masquerade" diff --git a/k3s/roles/server-setup/templates/ip-masquerade.sh.j2 b/k3s/roles/server-setup/templates/ip-masquerade.sh.j2 new file mode 100644 index 0000000..6ef2d0a --- /dev/null +++ b/k3s/roles/server-setup/templates/ip-masquerade.sh.j2 @@ -0,0 +1,4 @@ +#!/bin/bash +echo 1 > /proc/sys/net/ipv4/ip_forward +iptables -t nat -A POSTROUTING -s {{network_cidr}} -o eth0 -j MASQUERADE +