RomanNum3ral
/
k3s-ansible
mirror of https://github.com/timothystewart6/k3s-ansible.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(dependencies): upgrade supported cluster components 

- Bump K3s to v1.36.2+k3s1, Calico to v3.32.1, Cilium to v1.20.0,
  kube-vip to v1.2.2, kube-vip cloud provider to v0.0.12, and MetalLB to
  v0.16.0 across sample inventory, role defaults, and argument specs
- Pin the Cilium CLI with a new cilium_cli_tag (v0.19.7) instead of the
  floating stable.txt lookup
- Replace the CiliumBGPPeeringPolicy v2alpha1 BGP template with the
  v2 CiliumBGPClusterConfig, CiliumBGPPeerConfig, CiliumBGPAdvertisement,
  and CiliumLoadBalancerIPPool resource set
- Move Cilium load balancer Helm keys from bpf.loadBalancer to the valid
  top-level loadBalancer path
- Add preflight schema validation and remove the deprecated policy after
  the v2 objects are accepted
- Wait for cilium status after installation
- Pin kube-vip RBAC in a repository template instead of fetching a
  mutable URL, and include EndpointSlice permissions
- Fix the kube-vip bgppeers format to address:ASN comma-separated peers
- Fail clearly when the MetalLB speaker tag replacement does not apply
- Drop the obsolete MetalLB webhook service name version branch
This commit is contained in:
Timothy Stewart 2026-08-01 21:23:27 -05:00
parent f38d125b4d
commit ab3de49697
12 changed files with 221 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
inventory/sample/group_vars/all.yml
View File

@ -1,5 +1,5 @@
---
k3s_version: v1.30.2+k3s2
k3s_version: v1.36.2+k3s1
# this is the user that has ssh access to these machines
ansible_user: ansibleuser
systemd_dir: /etc/systemd/system
@ -13,13 +13,14 @@ flannel_iface: eth0
# uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about
# calico_iface: "eth0"
calico_ebpf: false # use eBPF dataplane instead of iptables
calico_tag: v3.28.0 # calico version tag
calico_tag: v3.32.1 # calico version tag
# uncomment cilium_iface to use cilium cni instead of flannel or calico
# ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel
# cilium_iface: "eth0"
cilium_mode: native # native when nodes on same subnet or using bgp, else set routed
cilium_tag: v1.16.0 # cilium version tag
cilium_tag: v1.20.0 # cilium version tag
cilium_cli_tag: v0.19.7 # cilium cli version tag
cilium_hubble: true # enable hubble observability relay and ui
# if using calico or cilium, you may specify the cluster pod cidr pool
@ -83,11 +84,11 @@ extra_server_args: >-
extra_agent_args: >-
{{ extra_args }}
# image tag for kube-vip
# image tag for kube-vi1.2
kube_vip_tag_version: v0.8.2
# tag for kube-vip-cloud-provider manifest
# kube_vip_cloud_provider_tag_version: "main"
# kube_vip_cloud_provider_tag_version: "v0.0.12"
# kube-vip ip range for load balancer
# (uncomment to use kube-vip for services instead of MetalLB)
@ -105,8 +106,8 @@ metal_lb_mode: layer2
# metal_lb_bgp_peer_address: "192.168.30.1"
# image tag for metal lb
metal_lb_speaker_tag_version: v0.14.8
metal_lb_controller_tag_version: v0.14.8
metal_lb_speaker_tag_version: v0.16.0
metal_lb_controller_tag_version: v0.16.0
# metallb ip range for load balancer
metal_lb_ip_range: 192.168.30.80-192.168.30.90

8
roles/k3s_server/defaults/main.yml
View File

@ -7,8 +7,8 @@ group_name_master: master
kube_vip_arp: true
kube_vip_iface:
kube_vip_cloud_provider_tag_version: main
kube_vip_tag_version: v0.7.2
kube_vip_cloud_provider_tag_version: v0.0.12
kube_vip_tag_version: v1.2.2
kube_vip_bgp: false
kube_vip_bgp_routerid: 127.0.0.1
@ -19,8 +19,8 @@ kube_vip_bgp_peeras: "64512"
kube_vip_bgp_peers: []
kube_vip_bgp_peers_groups: ['k3s_master']
metal_lb_controller_tag_version: v0.14.3
metal_lb_speaker_tag_version: v0.14.3
metal_lb_controller_tag_version: v0.16.0
metal_lb_speaker_tag_version: v0.16.0
metal_lb_type: native
retry_count: 20

8
roles/k3s_server/meta/main.yml
View File

@ -80,11 +80,11 @@ argument_specs:
kube_vip_tag_version:
description: Image tag for kube-vip
default: v0.7.2
default: v1.2.2
kube_vip_cloud_provider_tag_version:
description: Tag for kube-vip-cloud-provider manifest when enabled
default: main
default: v0.0.12
kube_vip_lb_ip_range:
description: IP range for kube-vip load balancer
@ -92,11 +92,11 @@ argument_specs:
metal_lb_controller_tag_version:
description: Image tag for MetalLB
default: v0.14.3
default: v0.16.0
metal_lb_speaker_tag_version:
description: Image tag for MetalLB
default: v0.14.3
default: v0.16.0
metal_lb_type:
choices:

19
roles/k3s_server/tasks/metallb.yml
View File

@ -28,3 +28,22 @@
loop_control:
label: "{{ item.change }} => {{ item.to }}"
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- name: Read back MetalLB manifest to verify the speaker image tag
ansible.builtin.set_fact:
metallb_manifest_has_speaker_tag: >-
{{
metal_lb_speaker_tag_version in
lookup('ansible.builtin.file', '/var/lib/rancher/k3s/server/manifests/metallb-crds.yaml')
}}
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- name: Fail if MetalLB speaker tag was not applied to the manifest
ansible.builtin.fail:
msg: >-
MetalLB speaker image tag {{ metal_lb_speaker_tag_version }}
was not found in the downloaded manifest. The upstream image
reference may have changed.
when:
- ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- not metallb_manifest_has_speaker_tag

6
roles/k3s_server/tasks/vip.yml
View File

@ -12,9 +12,9 @@
mode: "0644"
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- name: Download vip rbac manifest to first master
ansible.builtin.get_url:
url: https://kube-vip.io/manifests/rbac.yaml
- name: Copy kube-vip RBAC manifest to first master
ansible.builtin.template:
src: kube-vip-rbac.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/vip-rbac.yaml
owner: root
group: root

48
roles/k3s_server/templates/kube-vip-rbac.yaml.j2 Normal file
View File

@ -0,0 +1,48 @@
# Repository-owned kube-vip RBAC derived from the official v1.2.2
# `kube-vip manifest rbac` output (run with `--inCluster`).
# Source: https://github.com/kube-vip/kube-vip/blob/v1.2.2/pkg/kubevip/config_generator.go
# Permissions are kept exactly as broad as the upstream manifest.
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-vip
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
name: system:kube-vip-role
rules:
- apiGroups: [""]
resources: ["services/status"]
verbs: ["update"]
- apiGroups: [""]
resources: ["services", "endpoints"]
verbs: ["list", "get", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["list", "get", "watch", "update", "patch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["list", "get", "watch", "update", "create"]
- apiGroups: ["discovery.k8s.io"]
resources: ["endpointslices"]
verbs: ["list", "get", "watch", "update"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:kube-vip-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kube-vip-role
subjects:
- kind: ServiceAccount
name: kube-vip
namespace: kube-system

2
roles/k3s_server/templates/vip.yaml.j2
View File

@ -63,7 +63,7 @@ spec:
{% endif %}
{% if _kube_vip_bgp_peers | length > 0 %}
- name: bgppeers
value: "{{ _kube_vip_bgp_peers | map(attribute='peer_address') | zip(_kube_vip_bgp_peers| map(attribute='peer_asn')) | map('join', ',') | join(':') }}" # yamllint disable-line rule:line-length
value: "{{ _kube_vip_bgp_peers | map(attribute='peer_address') | zip(_kube_vip_bgp_peers | map(attribute='peer_asn')) | map('join', ':') | join(',') }}" # yamllint disable-line rule:line-length
{% else %}
{% if kube_vip_bgp_as is defined %}
- name: bgp_as

6
roles/k3s_server_post/defaults/main.yml
View File

@ -9,7 +9,7 @@ calico_ebpf: false
calico_encapsulation: VXLANCrossSubnet
calico_natOutgoing: Enabled # noqa var-naming
calico_nodeSelector: all() # noqa var-naming
calico_tag: v3.27.2
calico_tag: v3.32.1
cilium_bgp: false
cilium_exportPodCIDR: true # noqa var-naming
@ -20,6 +20,8 @@ cilium_bgp_neighbors_groups: ['k3s_all']
cilium_bgp_lb_cidr: 192.168.31.0/24
cilium_hubble: true
cilium_mode: native
cilium_tag: v1.20.0
cilium_cli_tag: v0.19.7
cluster_cidr: 10.52.0.0/16
enable_bpf_masquerade: true
@ -28,5 +30,5 @@ group_name_master: master
metal_lb_mode: layer2
metal_lb_available_timeout: 240s
metal_lb_controller_tag_version: v0.14.3
metal_lb_controller_tag_version: v0.16.0
metal_lb_ip_range: 192.168.30.80-192.168.30.90

12
roles/k3s_server_post/meta/main.yml
View File

@ -43,7 +43,7 @@ argument_specs:
calico_tag:
description: Calico version tag
default: v3.27.2
default: v3.32.1
cilium_bgp:
description:
@ -99,6 +99,14 @@ argument_specs:
- native
- routed
cilium_tag:
description: Cilium version tag
default: v1.20.0
cilium_cli_tag:
description: Cilium CLI version tag
default: v0.19.7
cluster_cidr:
description: Inner-cluster IP range
default: 10.52.0.0/16
@ -123,7 +131,7 @@ argument_specs:
metal_lb_available_timeout:
description: Wait for MetalLB resources
default: 240s
default: 240s6.0
metal_lb_ip_range:
description: MetalLB ip range for load balancer

65
roles/k3s_server_post/tasks/cilium.yml
View File

@ -30,26 +30,13 @@
}}
when: cilium_cli_installed.rc == 0
- name: Get latest stable Cilium CLI version file
ansible.builtin.get_url:
url: https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt
dest: /tmp/k3s/cilium-cli-stable.txt
owner: root
group: root
mode: "0755"
- name: Read Cilium CLI stable version from file
ansible.builtin.command: cat /tmp/k3s/cilium-cli-stable.txt
register: cli_ver
changed_when: false
- name: Log installed Cilium CLI version
ansible.builtin.debug:
msg: "Installed Cilium CLI version: {{ installed_cli_version | default('Not installed') }}"
- name: Log latest stable Cilium CLI version
- name: Log pinned Cilium CLI version
ansible.builtin.debug:
msg: "Latest Cilium CLI version: {{ cli_ver.stdout }}"
msg: "Pinned Cilium CLI version: {{ cilium_cli_tag }}"
- name: Determine if Cilium CLI needs installation or update
ansible.builtin.set_fact:
@ -57,7 +44,7 @@
{{
cilium_cli_installed.rc != 0 or
(cilium_cli_installed.rc == 0 and
installed_cli_version != cli_ver.stdout)
installed_cli_version != cilium_cli_tag)
}}
- name: Install or update Cilium CLI
@ -78,7 +65,7 @@
- .tar.gz
- .tar.gz.sha256sum
vars:
cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cli_ver.stdout }}
cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_tag }}
- name: Verify the downloaded tarball
ansible.builtin.shell: |
@ -181,8 +168,8 @@
--helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }}
--helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }}
{% if kube_proxy_replacement is not false %}
--helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm }}
--helm-set bpf.loadBalancer.mode={{ bpf_lb_mode }}
--helm-set loadBalancer.algorithm={{ bpf_lb_algorithm }}
--helm-set loadBalancer.mode={{ bpf_lb_mode }}
{% endif %}
environment:
KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
@ -218,6 +205,16 @@
when: >-
not item.check_hubble | default(false) or (item.check_hubble | default(false) and cilium_hubble)
- name: Wait for Cilium status to be healthy
ansible.builtin.command: cilium status --wait
environment:
KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
register: cilium_status
changed_when: false
until: cilium_status.rc == 0
retries: 30
delay: 7
- name: Configure Cilium BGP
when: cilium_bgp
block:
@ -233,23 +230,37 @@
group: root
mode: "0755"
- name: Preflight validate rendered BGP manifests against installed CRDs
ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }}
apply --dry-run=server -f /tmp/k3s/cilium-bgp.yaml
register: preflight_cr
changed_when: false
failed_when: "'error' in preflight_cr.stderr | lower"
- name: Apply BGP manifests
ansible.builtin.command:
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/cilium-bgp.yaml"
ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }}
apply -f /tmp/k3s/cilium-bgp.yaml
register: apply_cr
changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout"
failed_when: "'is invalid' in apply_cr.stderr"
ignore_errors: true
- name: Print error message if BGP manifests application fails
ansible.builtin.debug:
msg: "{{ apply_cr.stderr }}"
when: "'is invalid' in apply_cr.stderr"
- name: Remove deprecated CiliumBGPPeeringPolicy after v2 resources are accepted
ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }}
delete CiliumBGPPeeringPolicy.cilium.io 01-bgp-peering-policy
register: delete_old_policy
changed_when: "'deleted' in delete_old_policy.stdout"
failed_when: false
ignore_errors: true
- name: Test for BGP config resources
ansible.builtin.command: "{{ item }}"
loop:
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeeringPolicy.cilium.io"
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPClusterConfig.cilium.io"
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeerConfig.cilium.io"
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPAdvertisement.cilium.io"
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumLoadBalancerIPPool.cilium.io"
changed_when: false
loop_control:

14
roles/k3s_server_post/tasks/metallb.yml
View File

@ -85,17 +85,9 @@
- name: Set metallb webhook service name
ansible.builtin.set_fact:
metallb_webhook_service_name: >-
{{
(
(metal_lb_controller_tag_version | regex_replace('^v', ''))
is
version('0.14.4', '<', version_type='semver')
) | ternary(
'webhook-service',
'metallb-webhook-service'
)
}}
# Inspected the v0.16.0 manifest: the newer webhook Service name is used
# by every supported MetalLB release, so the old pre-0.14.4 branch is gone.
metallb_webhook_service_name: metallb-webhook-service
- name: Test metallb-system webhook-service endpoint
ansible.builtin.command: >-

114
roles/k3s_server_post/templates/cilium.crs.j2
View File

@ -1,40 +1,84 @@
apiVersion: "cilium.io/v2alpha1"
kind: CiliumBGPPeeringPolicy
# Cilium BGP Control Plane v2 resources.
# Replace the deprecated v2alpha1 CiliumBGPPeeringPolicy that was removed
# in Cilium 1.19.
{% set _cilium_default_peer = {'peer_address': cilium_bgp_peer_address, 'peer_asn': cilium_bgp_peer_asn} %}
{% set _cilium_peers = _cilium_bgp_neighbors if _cilium_bgp_neighbors | length > 0 else [_cilium_default_peer] %}
apiVersion: cilium.io/v2
kind: CiliumBGPPeerConfig
metadata:
name: 01-bgp-peering-policy
spec: # CiliumBGPPeeringPolicySpec
virtualRouters: # []CiliumBGPVirtualRouter
- localASN: {{ cilium_bgp_my_asn }}
exportPodCIDR: {{ cilium_exportPodCIDR | default('true') }}
neighbors: # []CiliumBGPNeighbor
{% if _cilium_bgp_neighbors | length > 0 %}
{% for item in _cilium_bgp_neighbors %}
- peerAddress: '{{ item.peer_address + "/32"}}'
peerASN: {{ item.peer_asn }}
eBGPMultihopTTL: 10
connectRetryTimeSeconds: 120
holdTimeSeconds: 90
keepAliveTimeSeconds: 30
gracefulRestart:
enabled: true
restartTimeSeconds: 120
{% endfor %}
{% else %}
- peerAddress: '{{ cilium_bgp_peer_address + "/32"}}'
peerASN: {{ cilium_bgp_peer_asn }}
eBGPMultihopTTL: 10
connectRetryTimeSeconds: 120
holdTimeSeconds: 90
keepAliveTimeSeconds: 30
gracefulRestart:
enabled: true
restartTimeSeconds: 120
{% endif %}
serviceSelector:
matchExpressions:
- {key: somekey, operator: NotIn, values: ['never-used-value']}
name: cilium-peer
spec:
# Matches the timers and multihop used by the previous v2alpha1 policy.
ebgpMultihop: 10
timers:
connectRetryTimeSeconds: 120
holdTimeSeconds: 90
keepAliveTimeSeconds: 30
gracefulRestart:
enabled: true
restartTimeSeconds: 120
families:
- afi: ipv4
safi: unicast
advertisements:
matchLabels:
advertise: "bgp"
---
apiVersion: "cilium.io/v2alpha1"
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
name: cilium-bgp
spec:
# Explicitly select every node so the BGP instance runs across the cluster.
nodeSelector:
matchExpressions:
- key: somekey
operator: NotIn
values: ['never-used-value']
bgpInstances:
- name: "instance-{{ cilium_bgp_my_asn }}"
localASN: {{ cilium_bgp_my_asn }}
peers:
{% for peer in _cilium_peers %}
- name: "peer-{{ peer.peer_asn }}-{{ loop.index }}"
peerASN: {{ peer.peer_asn }}
peerAddress: {{ peer.peer_address }}
peerConfigRef:
name: cilium-peer
{% endfor %}
{% if cilium_exportPodCIDR %}
---
apiVersion: cilium.io/v2
kind: CiliumBGPAdvertisement
metadata:
name: cilium-pod-cidrs
labels:
advertise: "bgp"
spec:
advertisements:
- advertisementType: "PodCIDR"
{% endif %}
---
apiVersion: cilium.io/v2
kind: CiliumBGPAdvertisement
metadata:
name: cilium-lb-services
labels:
advertise: "bgp"
spec:
advertisements:
- advertisementType: "Service"
service:
addresses:
- LoadBalancerIP
# Advertise all Services carrying an ingress address from the pool.
selector:
matchExpressions:
- key: somekey
operator: NotIn
values: ['never-used-value']
---
apiVersion: cilium.io/v2
kind: CiliumLoadBalancerIPPool
metadata:
name: "01-lb-pool"