From f7857cd5eb1467cd8c14fb7c05b089e551c62a62 Mon Sep 17 00:00:00 2001 From: Jens Heinitz Date: Thu, 2 Sep 2021 17:06:07 +0200 Subject: [PATCH] added roles and playbooks for k3s --- inventories/group_vars/k3s_cluster.yml | 7 ++ inventories/inventory_jens.ini | 12 +++ playbooks/k3s_reset.yml | 8 ++ playbooks/site.yml | 19 +++++ roles/download/tasks/main.yml | 36 +++++++++ roles/k3s/master/tasks/main.yml | 79 +++++++++++++++++++ roles/k3s/master/templates/k3s.service.j2 | 24 ++++++ roles/k3s/node/tasks/main.yml | 16 ++++ roles/k3s/node/templates/k3s.service.j2 | 24 ++++++ roles/k3s_reset/tasks/main.yml | 46 +++++++++++ .../k3s_reset/tasks/umount_with_children.yml | 16 ++++ roles/prereq/tasks/main.yml | 53 +++++++++++++ roles/raspberrypi/handlers/main.yml | 3 + roles/raspberrypi/tasks/main.yml | 51 ++++++++++++ roles/raspberrypi/tasks/prereq/CentOS.yml | 8 ++ roles/raspberrypi/tasks/prereq/Raspbian.yml | 25 ++++++ roles/raspberrypi/tasks/prereq/Ubuntu.yml | 8 ++ roles/raspberrypi/tasks/prereq/default.yml | 1 + 18 files changed, 436 insertions(+) create mode 100644 inventories/group_vars/k3s_cluster.yml create mode 100644 playbooks/k3s_reset.yml create mode 100644 playbooks/site.yml create mode 100644 roles/download/tasks/main.yml create mode 100644 roles/k3s/master/tasks/main.yml create mode 100644 roles/k3s/master/templates/k3s.service.j2 create mode 100644 roles/k3s/node/tasks/main.yml create mode 100644 roles/k3s/node/templates/k3s.service.j2 create mode 100644 roles/k3s_reset/tasks/main.yml create mode 100644 roles/k3s_reset/tasks/umount_with_children.yml create mode 100644 roles/prereq/tasks/main.yml create mode 100644 roles/raspberrypi/handlers/main.yml create mode 100644 roles/raspberrypi/tasks/main.yml create mode 100644 roles/raspberrypi/tasks/prereq/CentOS.yml create mode 100644 roles/raspberrypi/tasks/prereq/Raspbian.yml create mode 100644 roles/raspberrypi/tasks/prereq/Ubuntu.yml create mode 100644 roles/raspberrypi/tasks/prereq/default.yml diff --git a/inventories/group_vars/k3s_cluster.yml b/inventories/group_vars/k3s_cluster.yml new file mode 100644 index 0000000..62cc427 --- /dev/null +++ b/inventories/group_vars/k3s_cluster.yml @@ -0,0 +1,7 @@ +--- +k3s_version: v1.17.5+k3s1 +ansible_user: pi +systemd_dir: /etc/systemd/system +master_ip: "{{ hostvars[groups['k3s_master'][0]]['ansible_host'] | default(groups['k3s_master'][0]) }}" +extra_server_args: "" +extra_agent_args: "" diff --git a/inventories/inventory_jens.ini b/inventories/inventory_jens.ini index 2cc6258..e4e1d0d 100644 --- a/inventories/inventory_jens.ini +++ b/inventories/inventory_jens.ini @@ -2,6 +2,18 @@ #server1.seboto.net ansible_user=root server1.seboto.net +[k3s_master] +raspberrypi5.seboto.net + +[k3s_nodes] +raspberrypi2.seboto.net +raspberrypi3.seboto.net +raspberrypi4.seboto.net + +[k3s_cluster:children] +k3s_master +k3s_nodes + [alma] alma1.seboto.net diff --git a/playbooks/k3s_reset.yml b/playbooks/k3s_reset.yml new file mode 100644 index 0000000..8add4cc --- /dev/null +++ b/playbooks/k3s_reset.yml @@ -0,0 +1,8 @@ +--- + +- hosts: k3s_cluster + gather_facts: yes + remote_user: root + become: yes + roles: + - role: k3s_reset diff --git a/playbooks/site.yml b/playbooks/site.yml new file mode 100644 index 0000000..31cc96e --- /dev/null +++ b/playbooks/site.yml @@ -0,0 +1,19 @@ +--- + +- hosts: k3s_cluster + gather_facts: yes + become: yes + roles: + - role: prereq + - role: download + - role: raspberrypi + +- hosts: master + become: yes + roles: + - role: k3s/master + +- hosts: node + become: yes + roles: + - role: k3s/node diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml new file mode 100644 index 0000000..1450fd8 --- /dev/null +++ b/roles/download/tasks/main.yml @@ -0,0 +1,36 @@ +--- + +- name: Download k3s binary x64 + get_url: + url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s + checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt + dest: /usr/local/bin/k3s + owner: root + group: root + mode: 0755 + when: ansible_facts.architecture == "x86_64" + +- name: Download k3s binary arm64 + get_url: + url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-arm64 + checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm64.txt + dest: /usr/local/bin/k3s + owner: root + group: root + mode: 0755 + when: + - ( ansible_facts.architecture is search("arm") and + ansible_facts.userspace_bits == "64" ) or + ansible_facts.architecture is search("aarch64") + +- name: Download k3s binary armhf + get_url: + url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-armhf + checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm.txt + dest: /usr/local/bin/k3s + owner: root + group: root + mode: 0755 + when: + - ansible_facts.architecture is search("arm") + - ansible_facts.userspace_bits == "32" diff --git a/roles/k3s/master/tasks/main.yml b/roles/k3s/master/tasks/main.yml new file mode 100644 index 0000000..614986a --- /dev/null +++ b/roles/k3s/master/tasks/main.yml @@ -0,0 +1,79 @@ +--- + +- name: Copy K3s service file + register: k3s_service + template: + src: "k3s.service.j2" + dest: "{{ systemd_dir }}/k3s.service" + owner: root + group: root + mode: 0644 + +- name: Enable and check K3s service + systemd: + name: k3s + daemon_reload: yes + state: restarted + enabled: yes + +- name: Wait for node-token + wait_for: + path: /var/lib/rancher/k3s/server/node-token + +- name: Register node-token file access mode + stat: + path: /var/lib/rancher/k3s/server + register: p + +- name: Change file access node-token + file: + path: /var/lib/rancher/k3s/server + mode: "g+rx,o+rx" + +- name: Read node-token from master + slurp: + src: /var/lib/rancher/k3s/server/node-token + register: node_token + +- name: Store Master node-token + set_fact: + token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}" + +- name: Restore node-token file access + file: + path: /var/lib/rancher/k3s/server + mode: "{{ p.stat.mode }}" + +- name: Create directory .kube + file: + path: ~{{ ansible_user }}/.kube + state: directory + owner: "{{ ansible_user }}" + mode: "u=rwx,g=rx,o=" + +- name: Copy config file to user home directory + copy: + src: /etc/rancher/k3s/k3s.yaml + dest: ~{{ ansible_user }}/.kube/config + remote_src: yes + owner: "{{ ansible_user }}" + mode: "u=rw,g=,o=" + +- name: Replace https://localhost:6443 by https://master-ip:6443 + command: >- + k3s kubectl config set-cluster default + --server=https://{{ master_ip }}:6443 + --kubeconfig ~{{ ansible_user }}/.kube/config + changed_when: true + +- name: Create kubectl symlink + file: + src: /usr/local/bin/k3s + dest: /usr/local/bin/kubectl + state: link + +- name: Create crictl symlink + file: + src: /usr/local/bin/k3s + dest: /usr/local/bin/crictl + state: link diff --git a/roles/k3s/master/templates/k3s.service.j2 b/roles/k3s/master/templates/k3s.service.j2 new file mode 100644 index 0000000..ae5cb48 --- /dev/null +++ b/roles/k3s/master/templates/k3s.service.j2 @@ -0,0 +1,24 @@ +[Unit] +Description=Lightweight Kubernetes +Documentation=https://k3s.io +After=network-online.target + +[Service] +Type=notify +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s server {{ extra_server_args | default("") }} +KillMode=process +Delegate=yes +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/roles/k3s/node/tasks/main.yml b/roles/k3s/node/tasks/main.yml new file mode 100644 index 0000000..0ce8e08 --- /dev/null +++ b/roles/k3s/node/tasks/main.yml @@ -0,0 +1,16 @@ +--- + +- name: Copy K3s service file + template: + src: "k3s.service.j2" + dest: "{{ systemd_dir }}/k3s-node.service" + owner: root + group: root + mode: 0755 + +- name: Enable and check K3s service + systemd: + name: k3s-node + daemon_reload: yes + state: restarted + enabled: yes diff --git a/roles/k3s/node/templates/k3s.service.j2 b/roles/k3s/node/templates/k3s.service.j2 new file mode 100644 index 0000000..99a0ac3 --- /dev/null +++ b/roles/k3s/node/templates/k3s.service.j2 @@ -0,0 +1,24 @@ +[Unit] +Description=Lightweight Kubernetes +Documentation=https://k3s.io +After=network-online.target + +[Service] +Type=notify +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --token {{ hostvars[groups['master'][0]]['token'] }} {{ extra_agent_args | default("") }} +KillMode=process +Delegate=yes +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/roles/k3s_reset/tasks/main.yml b/roles/k3s_reset/tasks/main.yml new file mode 100644 index 0000000..fe363be --- /dev/null +++ b/roles/k3s_reset/tasks/main.yml @@ -0,0 +1,46 @@ +--- +- name: Disable services + systemd: + name: "{{ item }}" + state: stopped + enabled: no + failed_when: false + with_items: + - k3s + - k3s-node + - k3s-server + - k3s-agent + +- name: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" + register: pkill_containerd_shim_runc + command: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" + changed_when: "pkill_containerd_shim_runc.rc == 0" + failed_when: false + +- name: Umount k3s filesystems + include_tasks: umount_with_children.yml + with_items: + - /run/k3s + - /var/lib/kubelet + - /run/netns + - /var/lib/rancher/k3s + loop_control: + loop_var: mounted_fs + +- name: Remove service files, binaries and data + file: + name: "{{ item }}" + state: absent + with_items: + - /usr/local/bin/k3s + - "{{ systemd_dir }}/k3s.service" + - "{{ systemd_dir }}/k3s-node.service" + - "{{ systemd_dir }}/k3s-server.service" + - "{{ systemd_dir }}/k3s-agent.service" + - /etc/rancher/k3s + - /var/lib/kubelet + - /var/lib/rancher/k3s + +- name: daemon_reload + systemd: + daemon_reload: yes diff --git a/roles/k3s_reset/tasks/umount_with_children.yml b/roles/k3s_reset/tasks/umount_with_children.yml new file mode 100644 index 0000000..5883b70 --- /dev/null +++ b/roles/k3s_reset/tasks/umount_with_children.yml @@ -0,0 +1,16 @@ +--- +- name: Get the list of mounted filesystems + shell: set -o pipefail && cat /proc/mounts | awk '{ print $2}' | grep -E "^{{ mounted_fs }}" + register: get_mounted_filesystems + args: + executable: /bin/bash + failed_when: false + changed_when: get_mounted_filesystems.stdout | length > 0 + check_mode: false + +- name: Umount filesystem + mount: + path: "{{ item }}" + state: unmounted + with_items: + "{{ get_mounted_filesystems.stdout_lines | reverse | list }}" diff --git a/roles/prereq/tasks/main.yml b/roles/prereq/tasks/main.yml new file mode 100644 index 0000000..97617cf --- /dev/null +++ b/roles/prereq/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: Set SELinux to disabled state + selinux: + state: disabled + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux'] + +- name: Enable IPv4 forwarding + sysctl: + name: net.ipv4.ip_forward + value: "1" + state: present + reload: yes + +- name: Enable IPv6 forwarding + sysctl: + name: net.ipv6.conf.all.forwarding + value: "1" + state: present + reload: yes + +- name: Add br_netfilter to /etc/modules-load.d/ + copy: + content: "br_netfilter" + dest: /etc/modules-load.d/br_netfilter.conf + mode: "u=rw,g=,o=" + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux'] + +- name: Load br_netfilter + modprobe: + name: br_netfilter + state: present + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux'] + +- name: Set bridge-nf-call-iptables (just to be sure) + sysctl: + name: "{{ item }}" + value: "1" + state: present + reload: yes + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux'] + loop: + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-ip6tables + +- name: Add /usr/local/bin to sudo secure_path + lineinfile: + line: 'Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin' + regexp: "Defaults(\\s)*secure_path(\\s)*=" + state: present + insertafter: EOF + path: /etc/sudoers + validate: 'visudo -cf %s' + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux'] diff --git a/roles/raspberrypi/handlers/main.yml b/roles/raspberrypi/handlers/main.yml new file mode 100644 index 0000000..d25cf90 --- /dev/null +++ b/roles/raspberrypi/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: reboot + reboot: diff --git a/roles/raspberrypi/tasks/main.yml b/roles/raspberrypi/tasks/main.yml new file mode 100644 index 0000000..b80c91f --- /dev/null +++ b/roles/raspberrypi/tasks/main.yml @@ -0,0 +1,51 @@ +--- +- name: Test for raspberry pi /proc/cpuinfo + command: grep -E "Raspberry Pi|BCM2708|BCM2709|BCM2835|BCM2836" /proc/cpuinfo + register: grep_cpuinfo_raspberrypi + failed_when: false + changed_when: false + +- name: Test for raspberry pi /proc/device-tree/model + command: grep -E "Raspberry Pi" /proc/device-tree/model + register: grep_device_tree_model_raspberrypi + failed_when: false + changed_when: false + +- name: Set raspberry_pi fact to true + set_fact: + raspberry_pi: true + when: + grep_cpuinfo_raspberrypi.rc == 0 or grep_device_tree_model_raspberrypi.rc == 0 + +- name: Set detected_distribution to Raspbian + set_fact: + detected_distribution: Raspbian + when: > + raspberry_pi|default(false) and + ( ansible_facts.lsb.id|default("") == "Raspbian" or + ansible_facts.lsb.description|default("") is match("[Rr]aspbian.*") ) + +- name: Set detected_distribution to Raspbian (ARM64 on Debian Buster) + set_fact: + detected_distribution: Raspbian + when: + - ansible_facts.architecture is search("aarch64") + - raspberry_pi|default(false) + - ansible_facts.lsb.description|default("") is match("Debian.*buster") + +- name: Set detected_distribution_major_version + set_fact: + detected_distribution_major_version: "{{ ansible_facts.lsb.major_release }}" + when: + - detected_distribution | default("") == "Raspbian" + +- name: execute OS related tasks on the Raspberry Pi + include_tasks: "{{ item }}" + with_first_found: + - "prereq/{{ detected_distribution }}-{{ detected_distribution_major_version }}.yml" + - "prereq/{{ detected_distribution }}.yml" + - "prereq/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml" + - "prereq/{{ ansible_distribution }}.yml" + - "prereq/default.yml" + when: + - raspberry_pi|default(false) diff --git a/roles/raspberrypi/tasks/prereq/CentOS.yml b/roles/raspberrypi/tasks/prereq/CentOS.yml new file mode 100644 index 0000000..af83564 --- /dev/null +++ b/roles/raspberrypi/tasks/prereq/CentOS.yml @@ -0,0 +1,8 @@ +--- +- name: Enable cgroup via boot commandline if not already enabled for Centos + lineinfile: + path: /boot/cmdline.txt + backrefs: yes + regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$' + line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' + notify: reboot diff --git a/roles/raspberrypi/tasks/prereq/Raspbian.yml b/roles/raspberrypi/tasks/prereq/Raspbian.yml new file mode 100644 index 0000000..42bfe7d --- /dev/null +++ b/roles/raspberrypi/tasks/prereq/Raspbian.yml @@ -0,0 +1,25 @@ +--- +- name: Activating cgroup support + lineinfile: + path: /boot/cmdline.txt + regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$' + line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' + backrefs: true + notify: reboot + +- name: Flush iptables before changing to iptables-legacy + iptables: + flush: true + changed_when: false # iptables flush always returns changed + +- name: Changing to iptables-legacy + alternatives: + path: /usr/sbin/iptables-legacy + name: iptables + register: ip4_legacy + +- name: Changing to ip6tables-legacy + alternatives: + path: /usr/sbin/ip6tables-legacy + name: ip6tables + register: ip6_legacy diff --git a/roles/raspberrypi/tasks/prereq/Ubuntu.yml b/roles/raspberrypi/tasks/prereq/Ubuntu.yml new file mode 100644 index 0000000..742fc21 --- /dev/null +++ b/roles/raspberrypi/tasks/prereq/Ubuntu.yml @@ -0,0 +1,8 @@ +--- +- name: Enable cgroup via boot commandline if not already enabled for Ubuntu on a Raspberry Pi + lineinfile: + path: /boot/firmware/cmdline.txt + backrefs: yes + regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$' + line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' + notify: reboot diff --git a/roles/raspberrypi/tasks/prereq/default.yml b/roles/raspberrypi/tasks/prereq/default.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/raspberrypi/tasks/prereq/default.yml @@ -0,0 +1 @@ +---