diff --git a/README.md b/README.md index 27b4aab..5d17f9c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,11 @@ +# About this fork + +See [this issue](https://github.com/timothystewart6/k3s-ansible/issues/667) for where this comes from. This updates a few versions: +- k3s to v1.34.2+k3s1 +- Calico to v3.31.0 +- Cilium to v1.18.5 +- All playbooks have been modified to reflect this and are working. + # Automated build of HA k3s Cluster with `kube-vip` and MetalLB ![Fully Automated K3S etcd High Availability Install](https://img.youtube.com/vi/CbkEWcUZ7zM/0.jpg) @@ -16,8 +24,8 @@ If you want more context on how this works, see: Build a Kubernetes cluster using Ansible with k3s. The goal is easily install a HA Kubernetes cluster on machines running: -- [x] Debian (tested on version 11) -- [x] Ubuntu (tested on version 22.04) +- [x] Debian (tested on version 11 & 12) +- [x] Ubuntu (tested on version 22.04 & 24.04) - [x] Rocky (tested on version 9) on processor architecture: diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index 8ddeb05..dcf58c4 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -1,5 +1,5 @@ --- -k3s_version: v1.30.2+k3s2 +k3s_version: v1.34.2+k3s1 # this is the user that has ssh access to these machines ansible_user: ansibleuser systemd_dir: /etc/systemd/system @@ -13,13 +13,13 @@ flannel_iface: eth0 # uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about # calico_iface: "eth0" calico_ebpf: false # use eBPF dataplane instead of iptables -calico_tag: v3.28.0 # calico version tag +calico_tag: v3.31.0 # calico version tag # uncomment cilium_iface to use cilium cni instead of flannel or calico # ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel # cilium_iface: "eth0" cilium_mode: native # native when nodes on same subnet or using bgp, else set routed -cilium_tag: v1.16.0 # cilium version tag +cilium_tag: v1.18.5 # cilium version tag cilium_hubble: true # enable hubble observability relay and ui # if using calico or cilium, you may specify the cluster pod cidr pool diff --git a/roles/k3s_server/tasks/main.yml b/roles/k3s_server/tasks/main.yml index 8ebaad7..a48ac43 100644 --- a/roles/k3s_server/tasks/main.yml +++ b/roles/k3s_server/tasks/main.yml @@ -44,7 +44,7 @@ block: - name: Verify that all nodes actually joined (check k3s-init.service if this fails) ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length + cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/control-plane=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length register: nodes until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length retries: "{{ retry_count | default(20) }}" diff --git a/roles/k3s_server_post/tasks/calico.yml b/roles/k3s_server_post/tasks/calico.yml index 2a9302f..09d5662 100644 --- a/roles/k3s_server_post/tasks/calico.yml +++ b/roles/k3s_server_post/tasks/calico.yml @@ -1,7 +1,7 @@ --- - name: Deploy Calico to cluster - when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] run_once: true + when: inventory_hostname == groups[group_name_master | default('master')][0] block: - name: Create manifests directory on first master ansible.builtin.file: @@ -11,13 +11,22 @@ group: root mode: "0755" - - name: "Download to first master: manifest for Tigera Operator and Calico CRDs" + # Calico v3.30+ split: CRDs are no longer bundled in tigera-operator.yaml. + - name: Download Tigera operator CRDs (v3.30+) ansible.builtin.get_url: - url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/tigera-operator.yaml + url: "https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/operator-crds.yaml" + dest: /tmp/k3s/operator-crds.yaml + owner: root + group: root + mode: "0644" + + - name: Download Tigera operator manifest + ansible.builtin.get_url: + url: "https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/tigera-operator.yaml" dest: /tmp/k3s/tigera-operator.yaml owner: root group: root - mode: "0755" + mode: "0644" - name: Copy Calico custom resources manifest to first master ansible.builtin.template: @@ -25,55 +34,60 @@ dest: /tmp/k3s/custom-resources.yaml owner: root group: root - mode: "0755" + mode: "0644" - - name: Deploy or replace Tigera Operator - block: - - name: Deploy Tigera Operator - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} create -f /tmp/k3s/tigera-operator.yaml" - register: create_operator - changed_when: "'created' in create_operator.stdout" - failed_when: "'Error' in create_operator.stderr and 'already exists' not in create_operator.stderr" - rescue: - - name: Replace existing Tigera Operator - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} replace -f /tmp/k3s/tigera-operator.yaml" - register: replace_operator - changed_when: "'replaced' in replace_operator.stdout" - failed_when: "'Error' in replace_operator.stderr" + - name: Apply Tigera operator CRDs (server-side to avoid annotation size limit) + ansible.builtin.command: + cmd: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + apply --server-side --force-conflicts + -f /tmp/k3s/operator-crds.yaml + register: apply_operator_crds + changed_when: "'configured' in apply_operator_crds.stdout or 'created' in apply_operator_crds.stdout" + failed_when: apply_operator_crds.rc != 0 - - name: Wait for Tigera Operator resources + - name: Wait for Tigera operator CRDs to be established ansible.builtin.command: >- - {{ k3s_kubectl_binary | default('k3s kubectl') }} wait {{ item.type }}/{{ item.name }} - --namespace='tigera-operator' + {{ k3s_kubectl_binary | default('k3s kubectl') }} wait + --for=condition=Established + --timeout=180s + crd/{{ item }} + register: crd_wait + changed_when: false + until: crd_wait is succeeded + retries: 10 + delay: 6 + with_items: + # These cover the common objects referenced by calico custom-resources.yaml + - installations.operator.tigera.io + - apiservers.operator.tigera.io + - imagesets.operator.tigera.io + + - name: Apply Tigera Operator + ansible.builtin.command: + cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/tigera-operator.yaml" + register: apply_operator + changed_when: "'configured' in apply_operator.stdout or 'created' in apply_operator.stdout" + failed_when: apply_operator.rc != 0 + + - name: Wait for Tigera Operator deployment to be Available + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} wait deployment/tigera-operator + --namespace=tigera-operator --for=condition=Available=True - --timeout=30s + --timeout=300s register: tigera_result changed_when: false until: tigera_result is succeeded - retries: 7 - delay: 7 - with_items: - - { name: tigera-operator, type: deployment } - loop_control: - label: "{{ item.type }}/{{ item.name }}" + retries: 10 + delay: 6 - - name: Deploy Calico custom resources - block: - - name: Deploy custom resources for Calico - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} create -f /tmp/k3s/custom-resources.yaml" - register: create_cr - changed_when: "'created' in create_cr.stdout" - failed_when: "'Error' in create_cr.stderr and 'already exists' not in create_cr.stderr" - rescue: - - name: Apply new Calico custom resource manifest - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/custom-resources.yaml" - register: apply_cr - changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout" - failed_when: "'Error' in apply_cr.stderr" + - name: Apply Calico custom resources + ansible.builtin.command: + cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/custom-resources.yaml" + register: apply_cr + changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout" + failed_when: apply_cr.rc != 0 - name: Wait for Calico system resources to be available ansible.builtin.command: >- @@ -87,34 +101,28 @@ --namespace='{{ item.namespace }}' --for=condition=Available {% endif %} - --timeout=30s - register: cr_result + --timeout=180s + register: calico_wait changed_when: false - until: cr_result is succeeded - retries: 30 + until: calico_wait is succeeded + retries: 60 delay: 7 with_items: - { name: calico-typha, type: deployment, namespace: calico-system } - { name: calico-kube-controllers, type: deployment, namespace: calico-system } - - name: csi-node-driver - type: daemonset - selector: k8s-app=csi-node-driver - namespace: calico-system - - name: calico-node - type: daemonset - selector: k8s-app=calico-node - namespace: calico-system - - { name: calico-apiserver, type: deployment, namespace: calico-apiserver } + - { name: csi-node-driver, type: daemonset, selector: "k8s-app=csi-node-driver", namespace: calico-system } + - { name: calico-node, type: daemonset, selector: "k8s-app=calico-node", namespace: calico-system } + # - { name: calico-apiserver, type: deployment, namespace: calico-apiserver } loop_control: label: "{{ item.type }}/{{ item.name }}" - name: Patch Felix configuration for eBPF mode ansible.builtin.command: - cmd: > + cmd: >- {{ k3s_kubectl_binary | default('k3s kubectl') }} patch felixconfiguration default --type='merge' --patch='{"spec": {"bpfKubeProxyIptablesCleanupEnabled": false}}' register: patch_result - changed_when: "'felixconfiguration.projectcalico.org/default patched' in patch_result.stdout" - failed_when: "'Error' in patch_result.stderr" - when: calico_ebpf + changed_when: "'patched' in patch_result.stdout" + failed_when: patch_result.rc != 0 + when: calico_ebpf | default(false) | bool \ No newline at end of file