kube things

ansible/group_vars/kube.yml

@@ -0,0 +1,17 @@
+kube_release: "1.24"
+kube_apt: "{{ kube_release }}.2-00"
+containerd_config_default_write: true
+containerd_config_cgroup_driver_systemd: true
+
+docker_install_compose: false
+docker_users_obj: "{{ users|selectattr('docker', 'defined') }}"
+docker_users: "{{ docker_users_obj|map(attribute='username') }}"
+
+docker_apt_arch: "{{ apt_arch }}"
+
+docker_daemon_options:
+  log-driver: "journald"
+  log-opts: { "mode": "non-blocking" }
+  storage-driver: "fuse-overlayfs"
+  exec-opts: ["native.cgroupdriver=systemd"]
+  live-restore: true

ansible/requirements.yml

@@ -5,12 +5,7 @@ roles:
   - name: geerlingguy.ntp
   - name: geerlingguy.pip
   - name: geerlingguy.security
-  - name: alvistack.cri_o
-    version: "5.4.0"
-  - name: alvistack.runc
-  - name: alvistack.crun
-  - name: alvistack.containers_common
-  - name: alvistack.conmon
+  - name: geerlingguy.containerd
+    version: 1.3.0
 
-collections:
-  - name: sindhuparvathi_gopi.ansible_collection_template
+collections: []

ansible/roles/kubernetes/defaults/main.yml

@@ -1,4 +1,2 @@
-kube_release: "1.23"
-kube_apt: "{{ kube_release }}.5-00"
-cri_o_ver: "{{ kube_release }}:"
-cri_o_os: "x{{ ansible_distribution }}_{{ distribution_version }}"
+kube_release: "1.24"
+kube_apt: "{{ kube_release }}.1-00"

ansible/roles/kubernetes/handlers/main.yml

@@ -0,0 +1,33 @@
+- name: kube | systemctl restart systemd-modules-load.service
+  become: true
+  ansible.builtin.service:
+    name: "systemd-modules-load.service"
+    state: "restarted"
+  changed_when: false
+  failed_when: false
+
+- name: kube | docker options changed
+  become: true
+  ansible.builtin.service:
+    name: "docker.service"
+    state: "restarted"
+  changed_when: false
+  failed_when: false
+
+- name: kube | sysctl --system
+  become: true
+  ansible.builtin.command: sysctl --system
+  changed_when: false
+  failed_when: false
+
+- name: kube | boot options changed
+  become: true
+  changed_when: false
+  failed_when: false
+  ansible.builtin.reboot:
+    msg: "Reboot initiated by Ansible boot options changes"
+    connect_timeout: 5
+    reboot_timeout: 300
+    pre_reboot_delay: 0
+    post_reboot_delay: 30
+    test_command: uptime

ansible/roles/kubernetes/tasks/controlplane.yml

@@ -0,0 +1,28 @@
+- name: Install kubectl
+  become: true
+  ansible.builtin.apt:
+    name: "{{ item }}={{ kube_apt }}"
+    state: present
+  with_items:
+    - kubectl
+
+- name: Hold kubectl version
+  become: true
+  ansible.builtin.dpkg_selections:
+    name: "{{ item }}"
+    selection: "hold"
+  with_items:
+    - kubectl
+    - kubelet
+    - kubeadm
+
+- name: Add kubectl alias to bash
+  ansible.builtin.lineinfile:
+    mode: "0644"
+    path: /home/{{ users.0.username }}/.bashrc
+    line: "{{ item }}"
+    create: yes
+  with_items:
+    - source <(kubectl completion bash)
+    - alias k=kubectl
+    - complete -F __start_kubectl k

ansible/roles/kubernetes/tasks/main.yml

@@ -1,63 +1,6 @@
-- name: Disable SWAP
-  when: ansible_memory_mb.swap.total != 0
-  register: swap_disable
-  become: true
-  ansible.builtin.command: swapoff -a
+- name: Include node role
+  include_tasks: node.yml
 
-- name: Remove swapfile from /etc/fstab
-  become: true
-  ansible.posix.mount:
-    name: "{{ item }}"
-    fstype: swap
-    state: absent
-  with_items:
-    - swap
-
-- name: Set up kmsg in LXC # https://kevingoos.medium.com/kubernetes-inside-proxmox-lxc-cce5c9927942
-  when: inventory_hostname in groups['lxc_guest']
-  become: true
-  ansible.builtin.copy:
-    content: |
-      #!/bin/sh -e
-      if [ ! -e /dev/kmsg ]; then
-      ln -s /dev/console /dev/kmsg
-      fi
-      mount --make-rshared /
-    dest: /etc/rc.local
-    mode: "0755"
-
-- name: Add Apt signing key Google
-  become: true
-  ansible.builtin.apt_key:
-    url: "{{ item }}"
-    state: present
-  loop:
-    - https://packages.cloud.google.com/apt/doc/apt-key.gpg
-
-- name: Add repo for kubernetes
-  become: true
-  ansible.builtin.apt_repository:
-    filename: kubernetes
-    repo: "deb https://apt.kubernetes.io/ kubernetes-xenial main"
-    mode: "0666"
-    update_cache: yes
-
-- name: Install packages
-  become: true
-  ansible.builtin.apt:
-    name: "{{ item }}={{ kube_apt }}"
-    state: present
-  with_items:
-    - kubelet
-    - kubeadm
-    - kubectl
-
-- name: Hold kubernetes version
-  become: true
-  ansible.builtin.dpkg_selections:
-    name: "{{ item }}"
-    selection: "hold"
-  with_items:
-    - kubelet
-    - kubeadm
-    - kubectl
+- name: Include controlplane role
+  when: is_controlplane is defined
+  include_tasks: controlplane.yml

ansible/roles/kubernetes/tasks/node.yml

@@ -0,0 +1,154 @@
+- name: Disable SWAP
+  when: ansible_memory_mb.swap.total != 0
+  register: swap_disable
+  become: true
+  ansible.builtin.command: swapoff -a
+
+- name: Remove swapfile from /etc/fstab
+  become: true
+  ansible.posix.mount:
+    name: "{{ item }}"
+    fstype: swap
+    state: absent
+  with_items:
+    - swap
+
+- name: Add propogation to systemd
+  become: true
+  community.general.ini_file:
+    path: /lib/systemd/system/docker.service
+    section: Service
+    option: MountFlags
+    value: shared
+    mode: "0644"
+  notify:
+    - kube | docker options changed
+
+- name: Set up kmsg in LXC # https://kevingoos.medium.com/kubernetes-inside-proxmox-lxc-cce5c9927942
+  when: inventory_hostname in groups['lxc_guest']
+  become: true
+  ansible.builtin.copy:
+    content: |
+      #!/bin/sh -e
+      if [ ! -e /dev/kmsg ]; then
+        ln -s /dev/console /dev/kmsg
+      fi
+      mount --make-rshared /
+    dest: /etc/rc.local
+    mode: "0755"
+  notify:
+    - kube | boot options changed
+
+- name: Add cgroup directives to boot command line config
+  when: inventory_hostname in groups['raspberries']
+  become: yes
+  ansible.builtin.lineinfile:
+    path: /boot/firmware/cmdline.txt
+    regexp: '((.)+?)(\scgroup_\w+=\w+)*$'
+    line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
+    backrefs: yes
+  notify:
+    - kube | boot options changed
+
+- name: Set GPU memory split to 16 MB
+  when: inventory_hostname in groups['raspberries']
+  become: yes
+  community.general.ini_file:
+    path: /boot/firmware/config.txt
+    section: pi4
+    option: gpu_mem
+    value: 16
+    create: yes
+  notify:
+    - kube | boot options changed
+
+- name: Tweak modeprobe entries
+  become: true
+  ansible.builtin.ini_file:
+    option: "{{ item.option }}"
+    state: "{{ item.state }}"
+    path: "/usr/lib/modules-load.d/kube.conf"
+    section: ""
+    mode: "0644"
+    allow_no_value: true
+  loop:
+    - { state: "present", option: "br_netfilter" }
+    - { state: "present", option: "overlay" }
+  notify:
+    - kube | systemctl restart systemd-modules-load.service
+
+- name: Tweak sysctl entries
+  become: true
+  ansible.builtin.sysctl:
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    state: "{{ item.state }}"
+    sysctl_file: "/etc/sysctl.conf"
+    reload: false
+  loop:
+    - { state: "present", name: "kernel.pid_max", value: "4194303" }
+    - {
+        state: "present",
+        name: "net.bridge.bridge-nf-call-arptables",
+        value: "1",
+      }
+    - {
+        state: "present",
+        name: "net.bridge.bridge-nf-call-ip6tables",
+        value: "1",
+      }
+    - {
+        state: "present",
+        name: "net.bridge.bridge-nf-call-iptables",
+        value: "1",
+      }
+    - { state: "present", name: "net.ipv4.ip_forward", value: "1" }
+    - { state: "present", name: "net.ipv6.conf.all.disable_ipv6", value: "1" }
+    - { state: "present", name: "net.ipv6.conf.all.forwarding", value: "0" }
+    - {
+        state: "present",
+        name: "net.ipv6.conf.default.disable_ipv6",
+        value: "1",
+      }
+    - { state: "present", name: "net.ipv6.conf.lo.disable_ipv6", value: "1" }
+    - { state: "present", name: "vm.min_free_kbytes", value: "65536" }
+    - { state: "present", name: "vm.swappiness", value: "0" }
+  notify:
+    - kube | sysctl --system
+
+- name: Add Apt signing key for Google and Libcontainers
+  become: true
+  ansible.builtin.apt_key:
+    url: "{{ item }}"
+    state: present
+  loop:
+    - https://packages.cloud.google.com/apt/doc/apt-key.gpg
+
+- name: Add repo for Google and Libcontainers
+  become: true
+  ansible.builtin.apt_repository:
+    filename: Kubernetes
+    repo: "deb {{ item }}"
+    mode: "0666"
+    update_cache: yes
+  loop:
+    - https://apt.kubernetes.io/ kubernetes-xenial main
+
+- name: Install kubernetes packages
+  become: true
+  ansible.builtin.apt:
+    name: "{{ item }}={{ kube_apt }}"
+    state: present
+  with_items:
+    - kubelet
+    - kubeadm
+
+- name: Hold kubernetes version
+  become: true
+  when: ! is_controlplane is defined
+  ansible.builtin.dpkg_selections:
+    name: "{{ item }}"
+    selection: "hold"
+  with_items:
+    - kubelet
+    - kubeadm

ansible/run.yml

@@ -140,29 +140,23 @@
   vars_files:
     - "vars/vault.yml"
   tasks:
-    - name: Include Cri-O role
+    - name: Install runtime dependencies
+          become: true
+      ansible.builtin.apt:
+        name: "{{ item }}"
+        state: present
+      with_items:
+        - fuse-overlayfs
+        - nfs-common
+        - open-iscsi
+    - name: Include Containerd role
       include_role:
-        name: alvistack.cri_o
+        name: geerlingguy.containerd
         apply:
           become: true
-    - name: Include Runc role
+    - name: Include Docker role
       include_role:
-        name: alvistack.runc
-        apply:
-          become: true
-    - name: Include Crun role
-      include_role:
-        name: alvistack.crun
-        apply:
-          become: true
-    - name: Include containers_common role
-      include_role:
-        name: alvistack.containers_common
-        apply:
-          become: true
-    - name: Include conmon role
-      include_role:
-        name: alvistack.conmon
+        name: geerlingguy.docker
         apply:
           become: true
     - name: Include Kubernetes role

kube-lxc

@@ -0,0 +1,16 @@
+#kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
+
+arch: amd64
+cores: 2
+hostname: controlplane
+memory: 4096
+net0: name=eth0,bridge=vmbr0,firewall=1,gw=10.0.2.1,hwaddr=36:A9:18:B8:F7:2B,ip=10.0.2.5/32,tag=3,type=veth
+ostype: ubuntu
+rootfs: local-lvm:vm-105-disk-0,size=50G
+searchdomain: kube.rostvik.site
+swap: 0
+features: fuse=1,mount=nfs,nesting=1
+lxc.apparmor.profile: unconfined
+lxc.cap.drop:
+lxc.cgroup.devices.allow: a
+lxc.mount.auto: proc:rw sys:rw