diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index 8ddeb05..380f6f3 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -1,5 +1,5 @@ --- -k3s_version: v1.30.2+k3s2 +k3s_version: v1.36.2+k3s1 # this is the user that has ssh access to these machines ansible_user: ansibleuser systemd_dir: /etc/systemd/system @@ -13,13 +13,14 @@ flannel_iface: eth0 # uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about # calico_iface: "eth0" calico_ebpf: false # use eBPF dataplane instead of iptables -calico_tag: v3.28.0 # calico version tag +calico_tag: v3.32.1 # calico version tag # uncomment cilium_iface to use cilium cni instead of flannel or calico # ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel # cilium_iface: "eth0" cilium_mode: native # native when nodes on same subnet or using bgp, else set routed -cilium_tag: v1.16.0 # cilium version tag +cilium_tag: v1.20.0 # cilium version tag +cilium_cli_tag: v0.19.7 # cilium cli version tag cilium_hubble: true # enable hubble observability relay and ui # if using calico or cilium, you may specify the cluster pod cidr pool @@ -83,11 +84,11 @@ extra_server_args: >- extra_agent_args: >- {{ extra_args }} -# image tag for kube-vip +# image tag for kube-vi1.2 kube_vip_tag_version: v0.8.2 # tag for kube-vip-cloud-provider manifest -# kube_vip_cloud_provider_tag_version: "main" +# kube_vip_cloud_provider_tag_version: "v0.0.12" # kube-vip ip range for load balancer # (uncomment to use kube-vip for services instead of MetalLB) @@ -105,8 +106,8 @@ metal_lb_mode: layer2 # metal_lb_bgp_peer_address: "192.168.30.1" # image tag for metal lb -metal_lb_speaker_tag_version: v0.14.8 -metal_lb_controller_tag_version: v0.14.8 +metal_lb_speaker_tag_version: v0.16.0 +metal_lb_controller_tag_version: v0.16.0 # metallb ip range for load balancer metal_lb_ip_range: 192.168.30.80-192.168.30.90 diff --git a/roles/k3s_server/defaults/main.yml b/roles/k3s_server/defaults/main.yml index 1d18efd..8fe55fb 100644 --- a/roles/k3s_server/defaults/main.yml +++ b/roles/k3s_server/defaults/main.yml @@ -7,8 +7,8 @@ group_name_master: master kube_vip_arp: true kube_vip_iface: -kube_vip_cloud_provider_tag_version: main -kube_vip_tag_version: v0.7.2 +kube_vip_cloud_provider_tag_version: v0.0.12 +kube_vip_tag_version: v1.2.2 kube_vip_bgp: false kube_vip_bgp_routerid: 127.0.0.1 @@ -19,8 +19,8 @@ kube_vip_bgp_peeras: "64512" kube_vip_bgp_peers: [] kube_vip_bgp_peers_groups: ['k3s_master'] -metal_lb_controller_tag_version: v0.14.3 -metal_lb_speaker_tag_version: v0.14.3 +metal_lb_controller_tag_version: v0.16.0 +metal_lb_speaker_tag_version: v0.16.0 metal_lb_type: native retry_count: 20 diff --git a/roles/k3s_server/meta/main.yml b/roles/k3s_server/meta/main.yml index 7d9fbfd..7a5c0f7 100644 --- a/roles/k3s_server/meta/main.yml +++ b/roles/k3s_server/meta/main.yml @@ -80,11 +80,11 @@ argument_specs: kube_vip_tag_version: description: Image tag for kube-vip - default: v0.7.2 + default: v1.2.2 kube_vip_cloud_provider_tag_version: description: Tag for kube-vip-cloud-provider manifest when enabled - default: main + default: v0.0.12 kube_vip_lb_ip_range: description: IP range for kube-vip load balancer @@ -92,11 +92,11 @@ argument_specs: metal_lb_controller_tag_version: description: Image tag for MetalLB - default: v0.14.3 + default: v0.16.0 metal_lb_speaker_tag_version: description: Image tag for MetalLB - default: v0.14.3 + default: v0.16.0 metal_lb_type: choices: diff --git a/roles/k3s_server/tasks/metallb.yml b/roles/k3s_server/tasks/metallb.yml index 7624d16..ec4de88 100644 --- a/roles/k3s_server/tasks/metallb.yml +++ b/roles/k3s_server/tasks/metallb.yml @@ -28,3 +28,22 @@ loop_control: label: "{{ item.change }} => {{ item.to }}" when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] + +- name: Read back MetalLB manifest to verify the speaker image tag + ansible.builtin.set_fact: + metallb_manifest_has_speaker_tag: >- + {{ + metal_lb_speaker_tag_version in + lookup('ansible.builtin.file', '/var/lib/rancher/k3s/server/manifests/metallb-crds.yaml') + }} + when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] + +- name: Fail if MetalLB speaker tag was not applied to the manifest + ansible.builtin.fail: + msg: >- + MetalLB speaker image tag {{ metal_lb_speaker_tag_version }} + was not found in the downloaded manifest. The upstream image + reference may have changed. + when: + - ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] + - not metallb_manifest_has_speaker_tag diff --git a/roles/k3s_server/tasks/vip.yml b/roles/k3s_server/tasks/vip.yml index aba5b4f..857ed90 100644 --- a/roles/k3s_server/tasks/vip.yml +++ b/roles/k3s_server/tasks/vip.yml @@ -12,9 +12,9 @@ mode: "0644" when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] -- name: Download vip rbac manifest to first master - ansible.builtin.get_url: - url: https://kube-vip.io/manifests/rbac.yaml +- name: Copy kube-vip RBAC manifest to first master + ansible.builtin.template: + src: kube-vip-rbac.yaml.j2 dest: /var/lib/rancher/k3s/server/manifests/vip-rbac.yaml owner: root group: root diff --git a/roles/k3s_server/templates/kube-vip-rbac.yaml.j2 b/roles/k3s_server/templates/kube-vip-rbac.yaml.j2 new file mode 100644 index 0000000..1f15df2 --- /dev/null +++ b/roles/k3s_server/templates/kube-vip-rbac.yaml.j2 @@ -0,0 +1,48 @@ +# Repository-owned kube-vip RBAC derived from the official v1.2.2 +# `kube-vip manifest rbac` output (run with `--inCluster`). +# Source: https://github.com/kube-vip/kube-vip/blob/v1.2.2/pkg/kubevip/config_generator.go +# Permissions are kept exactly as broad as the upstream manifest. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list", "get", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "get", "watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list", "get", "watch", "update"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: + - kind: ServiceAccount + name: kube-vip + namespace: kube-system diff --git a/roles/k3s_server/templates/vip.yaml.j2 b/roles/k3s_server/templates/vip.yaml.j2 index 44469a6..a3a2fe3 100644 --- a/roles/k3s_server/templates/vip.yaml.j2 +++ b/roles/k3s_server/templates/vip.yaml.j2 @@ -63,7 +63,7 @@ spec: {% endif %} {% if _kube_vip_bgp_peers | length > 0 %} - name: bgppeers - value: "{{ _kube_vip_bgp_peers | map(attribute='peer_address') | zip(_kube_vip_bgp_peers| map(attribute='peer_asn')) | map('join', ',') | join(':') }}" # yamllint disable-line rule:line-length + value: "{{ _kube_vip_bgp_peers | map(attribute='peer_address') | zip(_kube_vip_bgp_peers | map(attribute='peer_asn')) | map('join', ':') | join(',') }}" # yamllint disable-line rule:line-length {% else %} {% if kube_vip_bgp_as is defined %} - name: bgp_as diff --git a/roles/k3s_server_post/defaults/main.yml b/roles/k3s_server_post/defaults/main.yml index 578e557..6ac138a 100644 --- a/roles/k3s_server_post/defaults/main.yml +++ b/roles/k3s_server_post/defaults/main.yml @@ -9,7 +9,7 @@ calico_ebpf: false calico_encapsulation: VXLANCrossSubnet calico_natOutgoing: Enabled # noqa var-naming calico_nodeSelector: all() # noqa var-naming -calico_tag: v3.27.2 +calico_tag: v3.32.1 cilium_bgp: false cilium_exportPodCIDR: true # noqa var-naming @@ -20,6 +20,8 @@ cilium_bgp_neighbors_groups: ['k3s_all'] cilium_bgp_lb_cidr: 192.168.31.0/24 cilium_hubble: true cilium_mode: native +cilium_tag: v1.20.0 +cilium_cli_tag: v0.19.7 cluster_cidr: 10.52.0.0/16 enable_bpf_masquerade: true @@ -28,5 +30,5 @@ group_name_master: master metal_lb_mode: layer2 metal_lb_available_timeout: 240s -metal_lb_controller_tag_version: v0.14.3 +metal_lb_controller_tag_version: v0.16.0 metal_lb_ip_range: 192.168.30.80-192.168.30.90 diff --git a/roles/k3s_server_post/meta/main.yml b/roles/k3s_server_post/meta/main.yml index f9fc83d..c41ba50 100644 --- a/roles/k3s_server_post/meta/main.yml +++ b/roles/k3s_server_post/meta/main.yml @@ -43,7 +43,7 @@ argument_specs: calico_tag: description: Calico version tag - default: v3.27.2 + default: v3.32.1 cilium_bgp: description: @@ -99,6 +99,14 @@ argument_specs: - native - routed + cilium_tag: + description: Cilium version tag + default: v1.20.0 + + cilium_cli_tag: + description: Cilium CLI version tag + default: v0.19.7 + cluster_cidr: description: Inner-cluster IP range default: 10.52.0.0/16 @@ -123,7 +131,7 @@ argument_specs: metal_lb_available_timeout: description: Wait for MetalLB resources - default: 240s + default: 240s6.0 metal_lb_ip_range: description: MetalLB ip range for load balancer diff --git a/roles/k3s_server_post/tasks/cilium.yml b/roles/k3s_server_post/tasks/cilium.yml index d7a48b0..735273a 100644 --- a/roles/k3s_server_post/tasks/cilium.yml +++ b/roles/k3s_server_post/tasks/cilium.yml @@ -30,26 +30,13 @@ }} when: cilium_cli_installed.rc == 0 - - name: Get latest stable Cilium CLI version file - ansible.builtin.get_url: - url: https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt - dest: /tmp/k3s/cilium-cli-stable.txt - owner: root - group: root - mode: "0755" - - - name: Read Cilium CLI stable version from file - ansible.builtin.command: cat /tmp/k3s/cilium-cli-stable.txt - register: cli_ver - changed_when: false - - name: Log installed Cilium CLI version ansible.builtin.debug: msg: "Installed Cilium CLI version: {{ installed_cli_version | default('Not installed') }}" - - name: Log latest stable Cilium CLI version + - name: Log pinned Cilium CLI version ansible.builtin.debug: - msg: "Latest Cilium CLI version: {{ cli_ver.stdout }}" + msg: "Pinned Cilium CLI version: {{ cilium_cli_tag }}" - name: Determine if Cilium CLI needs installation or update ansible.builtin.set_fact: @@ -57,7 +44,7 @@ {{ cilium_cli_installed.rc != 0 or (cilium_cli_installed.rc == 0 and - installed_cli_version != cli_ver.stdout) + installed_cli_version != cilium_cli_tag) }} - name: Install or update Cilium CLI @@ -78,7 +65,7 @@ - .tar.gz - .tar.gz.sha256sum vars: - cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cli_ver.stdout }} + cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_tag }} - name: Verify the downloaded tarball ansible.builtin.shell: | @@ -181,8 +168,8 @@ --helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }} --helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }} {% if kube_proxy_replacement is not false %} - --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm }} - --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode }} + --helm-set loadBalancer.algorithm={{ bpf_lb_algorithm }} + --helm-set loadBalancer.mode={{ bpf_lb_mode }} {% endif %} environment: KUBECONFIG: "{{ ansible_user_dir }}/.kube/config" @@ -218,6 +205,16 @@ when: >- not item.check_hubble | default(false) or (item.check_hubble | default(false) and cilium_hubble) + - name: Wait for Cilium status to be healthy + ansible.builtin.command: cilium status --wait + environment: + KUBECONFIG: "{{ ansible_user_dir }}/.kube/config" + register: cilium_status + changed_when: false + until: cilium_status.rc == 0 + retries: 30 + delay: 7 + - name: Configure Cilium BGP when: cilium_bgp block: @@ -233,23 +230,37 @@ group: root mode: "0755" + - name: Preflight validate rendered BGP manifests against installed CRDs + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + apply --dry-run=server -f /tmp/k3s/cilium-bgp.yaml + register: preflight_cr + changed_when: false + failed_when: "'error' in preflight_cr.stderr | lower" + - name: Apply BGP manifests - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/cilium-bgp.yaml" + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + apply -f /tmp/k3s/cilium-bgp.yaml register: apply_cr changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout" failed_when: "'is invalid' in apply_cr.stderr" - ignore_errors: true - - name: Print error message if BGP manifests application fails - ansible.builtin.debug: - msg: "{{ apply_cr.stderr }}" - when: "'is invalid' in apply_cr.stderr" + - name: Remove deprecated CiliumBGPPeeringPolicy after v2 resources are accepted + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + delete CiliumBGPPeeringPolicy.cilium.io 01-bgp-peering-policy + register: delete_old_policy + changed_when: "'deleted' in delete_old_policy.stdout" + failed_when: false + ignore_errors: true - name: Test for BGP config resources ansible.builtin.command: "{{ item }}" loop: - - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeeringPolicy.cilium.io" + - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPClusterConfig.cilium.io" + - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeerConfig.cilium.io" + - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPAdvertisement.cilium.io" - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumLoadBalancerIPPool.cilium.io" changed_when: false loop_control: diff --git a/roles/k3s_server_post/tasks/metallb.yml b/roles/k3s_server_post/tasks/metallb.yml index 4a3279c..63afcea 100644 --- a/roles/k3s_server_post/tasks/metallb.yml +++ b/roles/k3s_server_post/tasks/metallb.yml @@ -85,17 +85,9 @@ - name: Set metallb webhook service name ansible.builtin.set_fact: - metallb_webhook_service_name: >- - {{ - ( - (metal_lb_controller_tag_version | regex_replace('^v', '')) - is - version('0.14.4', '<', version_type='semver') - ) | ternary( - 'webhook-service', - 'metallb-webhook-service' - ) - }} + # Inspected the v0.16.0 manifest: the newer webhook Service name is used + # by every supported MetalLB release, so the old pre-0.14.4 branch is gone. + metallb_webhook_service_name: metallb-webhook-service - name: Test metallb-system webhook-service endpoint ansible.builtin.command: >- diff --git a/roles/k3s_server_post/templates/cilium.crs.j2 b/roles/k3s_server_post/templates/cilium.crs.j2 index 5a9e81c..c0cfbea 100644 --- a/roles/k3s_server_post/templates/cilium.crs.j2 +++ b/roles/k3s_server_post/templates/cilium.crs.j2 @@ -1,40 +1,84 @@ -apiVersion: "cilium.io/v2alpha1" -kind: CiliumBGPPeeringPolicy +# Cilium BGP Control Plane v2 resources. +# Replace the deprecated v2alpha1 CiliumBGPPeeringPolicy that was removed +# in Cilium 1.19. +{% set _cilium_default_peer = {'peer_address': cilium_bgp_peer_address, 'peer_asn': cilium_bgp_peer_asn} %} +{% set _cilium_peers = _cilium_bgp_neighbors if _cilium_bgp_neighbors | length > 0 else [_cilium_default_peer] %} +apiVersion: cilium.io/v2 +kind: CiliumBGPPeerConfig metadata: - name: 01-bgp-peering-policy -spec: # CiliumBGPPeeringPolicySpec - virtualRouters: # []CiliumBGPVirtualRouter - - localASN: {{ cilium_bgp_my_asn }} - exportPodCIDR: {{ cilium_exportPodCIDR | default('true') }} - neighbors: # []CiliumBGPNeighbor -{% if _cilium_bgp_neighbors | length > 0 %} -{% for item in _cilium_bgp_neighbors %} - - peerAddress: '{{ item.peer_address + "/32"}}' - peerASN: {{ item.peer_asn }} - eBGPMultihopTTL: 10 - connectRetryTimeSeconds: 120 - holdTimeSeconds: 90 - keepAliveTimeSeconds: 30 - gracefulRestart: - enabled: true - restartTimeSeconds: 120 -{% endfor %} -{% else %} - - peerAddress: '{{ cilium_bgp_peer_address + "/32"}}' - peerASN: {{ cilium_bgp_peer_asn }} - eBGPMultihopTTL: 10 - connectRetryTimeSeconds: 120 - holdTimeSeconds: 90 - keepAliveTimeSeconds: 30 - gracefulRestart: - enabled: true - restartTimeSeconds: 120 -{% endif %} - serviceSelector: - matchExpressions: - - {key: somekey, operator: NotIn, values: ['never-used-value']} + name: cilium-peer +spec: + # Matches the timers and multihop used by the previous v2alpha1 policy. + ebgpMultihop: 10 + timers: + connectRetryTimeSeconds: 120 + holdTimeSeconds: 90 + keepAliveTimeSeconds: 30 + gracefulRestart: + enabled: true + restartTimeSeconds: 120 + families: + - afi: ipv4 + safi: unicast + advertisements: + matchLabels: + advertise: "bgp" --- -apiVersion: "cilium.io/v2alpha1" +apiVersion: cilium.io/v2 +kind: CiliumBGPClusterConfig +metadata: + name: cilium-bgp +spec: + # Explicitly select every node so the BGP instance runs across the cluster. + nodeSelector: + matchExpressions: + - key: somekey + operator: NotIn + values: ['never-used-value'] + bgpInstances: + - name: "instance-{{ cilium_bgp_my_asn }}" + localASN: {{ cilium_bgp_my_asn }} + peers: +{% for peer in _cilium_peers %} + - name: "peer-{{ peer.peer_asn }}-{{ loop.index }}" + peerASN: {{ peer.peer_asn }} + peerAddress: {{ peer.peer_address }} + peerConfigRef: + name: cilium-peer +{% endfor %} +{% if cilium_exportPodCIDR %} +--- +apiVersion: cilium.io/v2 +kind: CiliumBGPAdvertisement +metadata: + name: cilium-pod-cidrs + labels: + advertise: "bgp" +spec: + advertisements: + - advertisementType: "PodCIDR" +{% endif %} +--- +apiVersion: cilium.io/v2 +kind: CiliumBGPAdvertisement +metadata: + name: cilium-lb-services + labels: + advertise: "bgp" +spec: + advertisements: + - advertisementType: "Service" + service: + addresses: + - LoadBalancerIP + # Advertise all Services carrying an ingress address from the pool. + selector: + matchExpressions: + - key: somekey + operator: NotIn + values: ['never-used-value'] +--- +apiVersion: cilium.io/v2 kind: CiliumLoadBalancerIPPool metadata: name: "01-lb-pool"