From d8aea6de29abe8ceff78e19de854383c40652eae Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 29 May 2023 09:38:06 -0400 Subject: [PATCH] feat: allow flux to take over cilium (#755) --- .config.sample.env | 21 +++-- .../inventory/group_vars/kubernetes/k3s.yml | 2 +- ansible/inventory/group_vars/master/k3s.yml | 4 +- ansible/playbooks/cluster-installation.yml | 72 ++++++++++----- ansible/playbooks/cluster-nuke.yml | 5 ++ ...aml.j2 => custom-cilium-helmchart.yaml.j2} | 2 +- configure | 7 ++ .../kube-system/cilium/app/helmrelease.yaml | 88 +++++++++++++++++++ .../kube-system/cilium/app/kustomization.yaml | 6 ++ kubernetes/apps/kube-system/cilium/ks.yaml | 16 ++++ .../apps/kube-system/kustomization.yaml | 1 + kubernetes/flux/repositories/helm/cilium.yaml | 9 ++ .../flux/repositories/helm/kustomization.yaml | 1 + tmpl/ansible/kube-vip.yml | 2 + tmpl/kubernetes/flux/cluster-settings.yaml | 3 + 15 files changed, 206 insertions(+), 33 deletions(-) rename ansible/playbooks/templates/{cilium-helmchart.yaml.j2 => custom-cilium-helmchart.yaml.j2} (94%) create mode 100644 kubernetes/apps/kube-system/cilium/app/helmrelease.yaml create mode 100644 kubernetes/apps/kube-system/cilium/app/kustomization.yaml create mode 100644 kubernetes/apps/kube-system/cilium/ks.yaml create mode 100644 kubernetes/flux/repositories/helm/cilium.yaml diff --git a/.config.sample.env b/.config.sample.env index b34e4fcabc6..b96300dbb0f 100644 --- a/.config.sample.env +++ b/.config.sample.env @@ -16,6 +16,14 @@ export BOOTSTRAP_FLUX_GITHUB_WEBHOOK_SECRET="generated" # NOTE: Must only contai # Set this to any other string and it will be used for the secret export BOOTSTRAP_WEAVE_GITOPS_ADMIN_PASSWORD="generated" # NOTE: Must only contain alphanumeric characters and dashes +# Age Public Key - string should start with age +# e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta +export BOOTSTRAP_AGE_PUBLIC_KEY="" + +# Choose your timezone +# e.g. America/New_York +export BOOTSTRAP_TIMEZONE="Etc/UTC" + # Choose one of your cloudflare domains # e.g. onedr0p.com export BOOTSTRAP_CLOUDFLARE_DOMAIN="" @@ -42,21 +50,16 @@ export BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR="" # that doesn't conflict with any other IP addresses here # e.g. 192.168.1.221 export BOOTSTRAP_METALLB_INGRESS_ADDR="" - -# Age Public Key - string should start with age -# e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta -export BOOTSTRAP_AGE_PUBLIC_KEY="" - # The IP Address to use with kube-vip # Pick a unused IP that is on the same network as your nodes # and outside the ${BOOTSTRAP_METALLB_LB_RANGE} range # and doesn't conflict with any other IP addresses here # e.g. 192.168.1.254 export BOOTSTRAP_KUBE_VIP_ADDR="" - -# Choose your timezone -# e.g. America/New_York -export BOOTSTRAP_TIMEZONE="Etc/UTC" +# Choose your cluster or service cidrs +# Leave this unchanged unless you know what you are doing +export BOOTSTRAP_CLUSTER_CIDR="10.42.0.0/16" +export BOOTSTRAP_SERVICE_CIDR="10.43.0.0/16" # # Ansible related variables diff --git a/ansible/inventory/group_vars/kubernetes/k3s.yml b/ansible/inventory/group_vars/kubernetes/k3s.yml index 802ce95fced..457e48a76c8 100644 --- a/ansible/inventory/group_vars/kubernetes/k3s.yml +++ b/ansible/inventory/group_vars/kubernetes/k3s.yml @@ -42,7 +42,7 @@ k3s_server_manifests_urls: # (list) A flat list of templates to deploy on the primary control plane node # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_templates: - - cilium-helmchart.yaml.j2 + - custom-cilium-helmchart.yaml.j2 # (list) A flat list of templates to deploy as static pods on all the control plane nodes # /var/lib/rancher/k3s/agent/pod-manifests diff --git a/ansible/inventory/group_vars/master/k3s.yml b/ansible/inventory/group_vars/master/k3s.yml index 48ed222e314..c8925462e00 100644 --- a/ansible/inventory/group_vars/master/k3s.yml +++ b/ansible/inventory/group_vars/master/k3s.yml @@ -29,9 +29,9 @@ k3s_server: disable-kube-proxy: true write-kubeconfig-mode: "644" # Network CIDR to use for pod IPs - cluster-cidr: "10.42.0.0/16" + cluster-cidr: "{{ cluster_cidr }}" # Network CIDR to use for service IPs - service-cidr: "10.43.0.0/16" + service-cidr: "{{ service_cidr }}" kube-controller-manager-arg: # Required to monitor kube-controller-manager with kube-prometheus-stack - "bind-address=0.0.0.0" diff --git a/ansible/playbooks/cluster-installation.yml b/ansible/playbooks/cluster-installation.yml index 8e813dcb8ff..b1bd6ec74f3 100644 --- a/ansible/playbooks/cluster-installation.yml +++ b/ansible/playbooks/cluster-installation.yml @@ -57,33 +57,65 @@ regexp: "https://127.0.0.1:6443" replace: "https://{{ k3s_registration_address }}:6443" - - name: Resource Readiness Check + # Unmanaging and removing the Cilium HelmChart is required for + # flux to take over managing the lifecycle of Cilium + + - name: Post installation of custom manifests tasks run_once: true - kubernetes.core.k8s_info: - kubeconfig: /etc/rancher/k3s/k3s.yaml - kind: "{{ item.kind }}" - name: "{{ item.name }}" - namespace: "{{ item.namespace | default('') }}" - wait: true - wait_sleep: 10 - wait_timeout: 360 - loop: - - name: cilium - kind: HelmChart - namespace: kube-system - - name: podmonitors.monitoring.coreos.com - kind: CustomResourceDefinition - - name: prometheusrules.monitoring.coreos.com - kind: CustomResourceDefinition - - name: servicemonitors.monitoring.coreos.com - kind: CustomResourceDefinition when: - k3s_server_manifests_templates | length > 0 or k3s_server_manifests_urls | length > 0 - k3s_control_node is defined - k3s_control_node + block: + - name: Wait for custom manifests to rollout + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + kind: "{{ item.kind }}" + name: "{{ item.name }}" + namespace: "{{ item.namespace | default('') }}" + wait: true + wait_sleep: 10 + wait_timeout: 360 + loop: + - name: cilium + kind: HelmChart + namespace: kube-system + - name: podmonitors.monitoring.coreos.com + kind: CustomResourceDefinition + - name: prometheusrules.monitoring.coreos.com + kind: CustomResourceDefinition + - name: servicemonitors.monitoring.coreos.com + kind: CustomResourceDefinition + - name: Wait for Cilium to rollout + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + kind: Job + name: helm-install-cilium + namespace: kube-system + wait: true + wait_condition: + type: Complete + status: true + wait_timeout: 360 + - name: Patch the Cilium HelmChart to unmanage it + kubernetes.core.k8s_json_patch: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: cilium + kind: HelmChart + namespace: kube-system + patch: + - op: add + path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged + value: "true" + - name: Remove the Cilium HelmChart CR + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: cilium + kind: HelmChart + namespace: kube-system + state: absent - # NOTE # Cleaning up certain manifests from the /var/lib/rancher/k3s/server/manifests directory # is needed because k3s has an awesome "feature" to always re-deploy them when the k3s # service is restarted. Removing them does not uninstall the manifests from your cluster. diff --git a/ansible/playbooks/cluster-nuke.yml b/ansible/playbooks/cluster-nuke.yml index e238e56a8df..5869d0b2bfc 100644 --- a/ansible/playbooks/cluster-nuke.yml +++ b/ansible/playbooks/cluster-nuke.yml @@ -22,6 +22,11 @@ ansible.builtin.pause: seconds: 5 tasks: + - name: Reboot + ansible.builtin.reboot: + msg: Rebooting nodes + reboot_timeout: 3600 + - name: Uninstall k3s ansible.builtin.include_role: name: xanmanning.k3s diff --git a/ansible/playbooks/templates/cilium-helmchart.yaml.j2 b/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 similarity index 94% rename from ansible/playbooks/templates/cilium-helmchart.yaml.j2 rename to ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 index 6d171f5350e..2c40f41a5b6 100644 --- a/ansible/playbooks/templates/cilium-helmchart.yaml.j2 +++ b/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 @@ -41,7 +41,7 @@ spec: ipam: mode: cluster-pool operator: - clusterPoolIPv4PodCIDRList: ["{{ k3s_server['cluster-cidr'] }}"] + clusterPoolIPv4PodCIDRList: ["{{ cluster_cidr }}"] clusterPoolIPv4MaskSize: 24 k8sServiceHost: "{{ kubevip_address }}" k8sServicePort: 6443 diff --git a/configure b/configure index 34eeb10319f..697ad873021 100755 --- a/configure +++ b/configure @@ -30,6 +30,7 @@ main() { verify_ansible_hosts verify_metallb verify_kubevip + verify_cluster_service_cidrs verify_addressing verify_age verify_git_repository @@ -252,6 +253,12 @@ verify_metallb() { _has_valid_ip "${BOOTSTRAP_METALLB_INGRESS_ADDR}" "BOOTSTRAP_METALLB_INGRESS_ADDR" } +verify_cluster_service_cidrs() { + #TODO: Validate input is a CIDR range + _has_envar "BOOTSTRAP_CLUSTER_CIDR" + _has_envar "BOOTSTRAP_SERVICE_CIDR" +} + verify_git_repository() { _has_envar "BOOTSTRAP_GIT_REPOSITORY" export GIT_TERMINAL_PROMPT=0 diff --git a/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml b/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml new file mode 100644 index 00000000000..f73915e1b70 --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml @@ -0,0 +1,88 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: cilium + namespace: kube-system +spec: + interval: 15m + chart: + spec: + chart: cilium + version: 1.13.2 + sourceRef: + kind: HelmRepository + name: cilium + namespace: flux-system + maxHistory: 2 + install: + createNamespace: true + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + bpf: + masquerade: true + cluster: + name: home-cluster + id: 1 + containerRuntime: + integration: containerd + socketPath: /var/run/k3s/containerd/containerd.sock + hubble: + enabled: true + metrics: + enabled: + - dns:query;ignoreAAAA + - drop + - tcp + - flow + - port-distribution + - icmp + - http + serviceMonitor: + enabled: true + relay: + enabled: true + rollOutPods: true + prometheus: + serviceMonitor: + enabled: true + ui: + enabled: true + rollOutPods: true + ingress: + enabled: true + className: nginx + hosts: + - &host "hubble.${SECRET_DOMAIN}" + tls: + - hosts: + - *host + ipam: + mode: cluster-pool + operator: + clusterPoolIPv4PodCIDRList: ["${CLUSTER_CIDR}"] + clusterPoolIPv4MaskSize: 24 + k8sServiceHost: "${KUBE_VIP_ADDR}" + k8sServicePort: 6443 + kubeProxyReplacement: strict + kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + operator: + enabled: true + replicas: 1 + rollOutPods: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + rollOutCiliumPods: true diff --git a/kubernetes/apps/kube-system/cilium/app/kustomization.yaml b/kubernetes/apps/kube-system/cilium/app/kustomization.yaml new file mode 100644 index 00000000000..1c3fdb04d23 --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kube-system +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/kube-system/cilium/ks.yaml b/kubernetes/apps/kube-system/cilium/ks.yaml new file mode 100644 index 00000000000..2efa949af3e --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/ks.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-apps-cilium + namespace: flux-system +spec: + path: ./kubernetes/apps/kube-system/cilium/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/apps/kube-system/kustomization.yaml b/kubernetes/apps/kube-system/kustomization.yaml index 9981ad8c1bf..626924aa9e9 100644 --- a/kubernetes/apps/kube-system/kustomization.yaml +++ b/kubernetes/apps/kube-system/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml + - ./cilium/ks.yaml - ./local-path-provisioner/ks.yaml - ./metrics-server/ks.yaml - ./reloader/ks.yaml diff --git a/kubernetes/flux/repositories/helm/cilium.yaml b/kubernetes/flux/repositories/helm/cilium.yaml new file mode 100644 index 00000000000..51c65d69178 --- /dev/null +++ b/kubernetes/flux/repositories/helm/cilium.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: cilium + namespace: flux-system +spec: + interval: 1h + url: https://helm.cilium.io diff --git a/kubernetes/flux/repositories/helm/kustomization.yaml b/kubernetes/flux/repositories/helm/kustomization.yaml index 93f0ab4b9e2..818532b3b9c 100644 --- a/kubernetes/flux/repositories/helm/kustomization.yaml +++ b/kubernetes/flux/repositories/helm/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - ./bitnami.yaml - ./bjw-s.yaml + - ./cilium.yaml - ./external-dns.yaml - ./hajimari.yaml - ./ingress-nginx.yaml diff --git a/tmpl/ansible/kube-vip.yml b/tmpl/ansible/kube-vip.yml index 0720e05e13a..fbfd042a2d6 100644 --- a/tmpl/ansible/kube-vip.yml +++ b/tmpl/ansible/kube-vip.yml @@ -1,3 +1,5 @@ --- # (string) The ARP address kube-vip broadcasts kubevip_address: "${BOOTSTRAP_KUBE_VIP_ADDR}" +cluster_cidr: "${BOOTSTRAP_CLUSTER_CIDR}" +service_cidr: "${BOOTSTRAP_SERVICE_CIDR}" diff --git a/tmpl/kubernetes/flux/cluster-settings.yaml b/tmpl/kubernetes/flux/cluster-settings.yaml index 279ac4ba5d4..ee04dd13b86 100644 --- a/tmpl/kubernetes/flux/cluster-settings.yaml +++ b/tmpl/kubernetes/flux/cluster-settings.yaml @@ -9,3 +9,6 @@ data: METALLB_LB_RANGE: "${BOOTSTRAP_METALLB_LB_RANGE}" METALLB_K8S_GATEWAY_ADDR: "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}" METALLB_INGRESS_ADDR: "${BOOTSTRAP_METALLB_INGRESS_ADDR}" + KUBE_VIP_ADDR: "${BOOTSTRAP_KUBE_VIP_ADDR}" + CLUSTER_CIDR: "${BOOTSTRAP_CLUSTER_CIDR}" + SERVICE_CIDR: "${BOOTSTRAP_SERVICE_CIDR}"