From 0ea54f4ffe9079acd09816a622e54a96c96fed2e Mon Sep 17 00:00:00 2001 From: Roxedus Date: Tue, 18 Oct 2022 22:18:54 +0200 Subject: [PATCH] kube things --- ansible/group_vars/kube.yml | 17 ++ ansible/requirements.yml | 11 +- ansible/roles/kubernetes/defaults/main.yml | 6 +- ansible/roles/kubernetes/handlers/main.yml | 33 ++++ .../roles/kubernetes/tasks/controlplane.yml | 28 ++++ ansible/roles/kubernetes/tasks/main.yml | 67 +------- ansible/roles/kubernetes/tasks/node.yml | 154 ++++++++++++++++++ ansible/run.yml | 32 ++-- kube-lxc | 16 ++ 9 files changed, 271 insertions(+), 93 deletions(-) create mode 100644 ansible/group_vars/kube.yml create mode 100644 ansible/roles/kubernetes/handlers/main.yml create mode 100644 ansible/roles/kubernetes/tasks/controlplane.yml create mode 100644 ansible/roles/kubernetes/tasks/node.yml create mode 100644 kube-lxc diff --git a/ansible/group_vars/kube.yml b/ansible/group_vars/kube.yml new file mode 100644 index 0000000..037eebf --- /dev/null +++ b/ansible/group_vars/kube.yml @@ -0,0 +1,17 @@ +kube_release: "1.24" +kube_apt: "{{ kube_release }}.2-00" +containerd_config_default_write: true +containerd_config_cgroup_driver_systemd: true + +docker_install_compose: false +docker_users_obj: "{{ users|selectattr('docker', 'defined') }}" +docker_users: "{{ docker_users_obj|map(attribute='username') }}" + +docker_apt_arch: "{{ apt_arch }}" + +docker_daemon_options: + log-driver: "journald" + log-opts: { "mode": "non-blocking" } + storage-driver: "fuse-overlayfs" + exec-opts: ["native.cgroupdriver=systemd"] + live-restore: true diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 4c66785..6ec3fa2 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -5,12 +5,7 @@ roles: - name: geerlingguy.ntp - name: geerlingguy.pip - name: geerlingguy.security - - name: alvistack.cri_o - version: "5.4.0" - - name: alvistack.runc - - name: alvistack.crun - - name: alvistack.containers_common - - name: alvistack.conmon + - name: geerlingguy.containerd + version: 1.3.0 -collections: - - name: sindhuparvathi_gopi.ansible_collection_template +collections: [] diff --git a/ansible/roles/kubernetes/defaults/main.yml b/ansible/roles/kubernetes/defaults/main.yml index 22e1607..430b7bc 100644 --- a/ansible/roles/kubernetes/defaults/main.yml +++ b/ansible/roles/kubernetes/defaults/main.yml @@ -1,4 +1,2 @@ -kube_release: "1.23" -kube_apt: "{{ kube_release }}.5-00" -cri_o_ver: "{{ kube_release }}:" -cri_o_os: "x{{ ansible_distribution }}_{{ distribution_version }}" +kube_release: "1.24" +kube_apt: "{{ kube_release }}.1-00" diff --git a/ansible/roles/kubernetes/handlers/main.yml b/ansible/roles/kubernetes/handlers/main.yml new file mode 100644 index 0000000..8c8cdf6 --- /dev/null +++ b/ansible/roles/kubernetes/handlers/main.yml @@ -0,0 +1,33 @@ +- name: kube | systemctl restart systemd-modules-load.service + become: true + ansible.builtin.service: + name: "systemd-modules-load.service" + state: "restarted" + changed_when: false + failed_when: false + +- name: kube | docker options changed + become: true + ansible.builtin.service: + name: "docker.service" + state: "restarted" + changed_when: false + failed_when: false + +- name: kube | sysctl --system + become: true + ansible.builtin.command: sysctl --system + changed_when: false + failed_when: false + +- name: kube | boot options changed + become: true + changed_when: false + failed_when: false + ansible.builtin.reboot: + msg: "Reboot initiated by Ansible boot options changes" + connect_timeout: 5 + reboot_timeout: 300 + pre_reboot_delay: 0 + post_reboot_delay: 30 + test_command: uptime diff --git a/ansible/roles/kubernetes/tasks/controlplane.yml b/ansible/roles/kubernetes/tasks/controlplane.yml new file mode 100644 index 0000000..8c6dae2 --- /dev/null +++ b/ansible/roles/kubernetes/tasks/controlplane.yml @@ -0,0 +1,28 @@ +- name: Install kubectl + become: true + ansible.builtin.apt: + name: "{{ item }}={{ kube_apt }}" + state: present + with_items: + - kubectl + +- name: Hold kubectl version + become: true + ansible.builtin.dpkg_selections: + name: "{{ item }}" + selection: "hold" + with_items: + - kubectl + - kubelet + - kubeadm + +- name: Add kubectl alias to bash + ansible.builtin.lineinfile: + mode: "0644" + path: /home/{{ users.0.username }}/.bashrc + line: "{{ item }}" + create: yes + with_items: + - source <(kubectl completion bash) + - alias k=kubectl + - complete -F __start_kubectl k diff --git a/ansible/roles/kubernetes/tasks/main.yml b/ansible/roles/kubernetes/tasks/main.yml index fe14fd2..c0a276c 100644 --- a/ansible/roles/kubernetes/tasks/main.yml +++ b/ansible/roles/kubernetes/tasks/main.yml @@ -1,63 +1,6 @@ -- name: Disable SWAP - when: ansible_memory_mb.swap.total != 0 - register: swap_disable - become: true - ansible.builtin.command: swapoff -a +- name: Include node role + include_tasks: node.yml -- name: Remove swapfile from /etc/fstab - become: true - ansible.posix.mount: - name: "{{ item }}" - fstype: swap - state: absent - with_items: - - swap - -- name: Set up kmsg in LXC # https://kevingoos.medium.com/kubernetes-inside-proxmox-lxc-cce5c9927942 - when: inventory_hostname in groups['lxc_guest'] - become: true - ansible.builtin.copy: - content: | - #!/bin/sh -e - if [ ! -e /dev/kmsg ]; then - ln -s /dev/console /dev/kmsg - fi - mount --make-rshared / - dest: /etc/rc.local - mode: "0755" - -- name: Add Apt signing key Google - become: true - ansible.builtin.apt_key: - url: "{{ item }}" - state: present - loop: - - https://packages.cloud.google.com/apt/doc/apt-key.gpg - -- name: Add repo for kubernetes - become: true - ansible.builtin.apt_repository: - filename: kubernetes - repo: "deb https://apt.kubernetes.io/ kubernetes-xenial main" - mode: "0666" - update_cache: yes - -- name: Install packages - become: true - ansible.builtin.apt: - name: "{{ item }}={{ kube_apt }}" - state: present - with_items: - - kubelet - - kubeadm - - kubectl - -- name: Hold kubernetes version - become: true - ansible.builtin.dpkg_selections: - name: "{{ item }}" - selection: "hold" - with_items: - - kubelet - - kubeadm - - kubectl +- name: Include controlplane role + when: is_controlplane is defined + include_tasks: controlplane.yml diff --git a/ansible/roles/kubernetes/tasks/node.yml b/ansible/roles/kubernetes/tasks/node.yml new file mode 100644 index 0000000..b75ebac --- /dev/null +++ b/ansible/roles/kubernetes/tasks/node.yml @@ -0,0 +1,154 @@ +- name: Disable SWAP + when: ansible_memory_mb.swap.total != 0 + register: swap_disable + become: true + ansible.builtin.command: swapoff -a + +- name: Remove swapfile from /etc/fstab + become: true + ansible.posix.mount: + name: "{{ item }}" + fstype: swap + state: absent + with_items: + - swap + +- name: Add propogation to systemd + become: true + community.general.ini_file: + path: /lib/systemd/system/docker.service + section: Service + option: MountFlags + value: shared + mode: "0644" + notify: + - kube | docker options changed + +- name: Set up kmsg in LXC # https://kevingoos.medium.com/kubernetes-inside-proxmox-lxc-cce5c9927942 + when: inventory_hostname in groups['lxc_guest'] + become: true + ansible.builtin.copy: + content: | + #!/bin/sh -e + if [ ! -e /dev/kmsg ]; then + ln -s /dev/console /dev/kmsg + fi + mount --make-rshared / + dest: /etc/rc.local + mode: "0755" + notify: + - kube | boot options changed + +- name: Add cgroup directives to boot command line config + when: inventory_hostname in groups['raspberries'] + become: yes + ansible.builtin.lineinfile: + path: /boot/firmware/cmdline.txt + regexp: '((.)+?)(\scgroup_\w+=\w+)*$' + line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' + backrefs: yes + notify: + - kube | boot options changed + +- name: Set GPU memory split to 16 MB + when: inventory_hostname in groups['raspberries'] + become: yes + community.general.ini_file: + path: /boot/firmware/config.txt + section: pi4 + option: gpu_mem + value: 16 + create: yes + notify: + - kube | boot options changed + +- name: Tweak modeprobe entries + become: true + ansible.builtin.ini_file: + option: "{{ item.option }}" + state: "{{ item.state }}" + path: "/usr/lib/modules-load.d/kube.conf" + section: "" + mode: "0644" + allow_no_value: true + loop: + - { state: "present", option: "br_netfilter" } + - { state: "present", option: "overlay" } + notify: + - kube | systemctl restart systemd-modules-load.service + +- name: Tweak sysctl entries + become: true + ansible.builtin.sysctl: + name: "{{ item.name }}" + value: "{{ item.value }}" + state: "{{ item.state }}" + sysctl_file: "/etc/sysctl.conf" + reload: false + loop: + - { state: "present", name: "kernel.pid_max", value: "4194303" } + - { + state: "present", + name: "net.bridge.bridge-nf-call-arptables", + value: "1", + } + - { + state: "present", + name: "net.bridge.bridge-nf-call-ip6tables", + value: "1", + } + - { + state: "present", + name: "net.bridge.bridge-nf-call-iptables", + value: "1", + } + - { state: "present", name: "net.ipv4.ip_forward", value: "1" } + - { state: "present", name: "net.ipv6.conf.all.disable_ipv6", value: "1" } + - { state: "present", name: "net.ipv6.conf.all.forwarding", value: "0" } + - { + state: "present", + name: "net.ipv6.conf.default.disable_ipv6", + value: "1", + } + - { state: "present", name: "net.ipv6.conf.lo.disable_ipv6", value: "1" } + - { state: "present", name: "vm.min_free_kbytes", value: "65536" } + - { state: "present", name: "vm.swappiness", value: "0" } + notify: + - kube | sysctl --system + +- name: Add Apt signing key for Google and Libcontainers + become: true + ansible.builtin.apt_key: + url: "{{ item }}" + state: present + loop: + - https://packages.cloud.google.com/apt/doc/apt-key.gpg + +- name: Add repo for Google and Libcontainers + become: true + ansible.builtin.apt_repository: + filename: Kubernetes + repo: "deb {{ item }}" + mode: "0666" + update_cache: yes + loop: + - https://apt.kubernetes.io/ kubernetes-xenial main + +- name: Install kubernetes packages + become: true + ansible.builtin.apt: + name: "{{ item }}={{ kube_apt }}" + state: present + with_items: + - kubelet + - kubeadm + +- name: Hold kubernetes version + become: true + when: ! is_controlplane is defined + ansible.builtin.dpkg_selections: + name: "{{ item }}" + selection: "hold" + with_items: + - kubelet + - kubeadm diff --git a/ansible/run.yml b/ansible/run.yml index 269974b..5d2eb82 100644 --- a/ansible/run.yml +++ b/ansible/run.yml @@ -140,29 +140,23 @@ vars_files: - "vars/vault.yml" tasks: - - name: Include Cri-O role + - name: Install runtime dependencies + become: true + ansible.builtin.apt: + name: "{{ item }}" + state: present + with_items: + - fuse-overlayfs + - nfs-common + - open-iscsi + - name: Include Containerd role include_role: - name: alvistack.cri_o + name: geerlingguy.containerd apply: become: true - - name: Include Runc role + - name: Include Docker role include_role: - name: alvistack.runc - apply: - become: true - - name: Include Crun role - include_role: - name: alvistack.crun - apply: - become: true - - name: Include containers_common role - include_role: - name: alvistack.containers_common - apply: - become: true - - name: Include conmon role - include_role: - name: alvistack.conmon + name: geerlingguy.docker apply: become: true - name: Include Kubernetes role diff --git a/kube-lxc b/kube-lxc new file mode 100644 index 0000000..fe8ba9c --- /dev/null +++ b/kube-lxc @@ -0,0 +1,16 @@ +#kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml + +arch: amd64 +cores: 2 +hostname: controlplane +memory: 4096 +net0: name=eth0,bridge=vmbr0,firewall=1,gw=10.0.2.1,hwaddr=36:A9:18:B8:F7:2B,ip=10.0.2.5/32,tag=3,type=veth +ostype: ubuntu +rootfs: local-lvm:vm-105-disk-0,size=50G +searchdomain: kube.rostvik.site +swap: 0 +features: fuse=1,mount=nfs,nesting=1 +lxc.apparmor.profile: unconfined +lxc.cap.drop: +lxc.cgroup.devices.allow: a +lxc.mount.auto: proc:rw sys:rw