diff --git a/.config.sample.env b/.config.sample.env index 6d233c185b8..b34e4fcabc6 100644 --- a/.config.sample.env +++ b/.config.sample.env @@ -21,9 +21,16 @@ export BOOTSTRAP_WEAVE_GITOPS_ADMIN_PASSWORD="generated" # NOTE: Must only conta export BOOTSTRAP_CLOUDFLARE_DOMAIN="" # The email you use to sign into Cloudflare with export BOOTSTRAP_CLOUDFLARE_EMAIL="" -# Your global Cloudflare API Key +# Your global Cloudflare API Key (not API token) export BOOTSTRAP_CLOUDFLARE_APIKEY="" +# Create a cloudflare tunnel to automatically ingress into your cluster +# https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/local/ +# cloudflared tunnel create +export BOOTSTRAP_CLOUDFLARE_ACCOUNT_TAG="" +export BOOTSTRAP_CLOUDFLARE_TUNNEL_SECRET="" +export BOOTSTRAP_CLOUDFLARE_TUNNEL_ID="" + # Pick a range of unused IPs that are on the same network as your nodes # You don't need many IPs, just choose 10 IPs to start with # e.g. 192.168.1.220-192.168.1.230 diff --git a/.github/labeler.yaml b/.github/labeler.yaml index 4ce4fcbabdb..6661f2ae312 100644 --- a/.github/labeler.yaml +++ b/.github/labeler.yaml @@ -5,7 +5,5 @@ area/github: - ".github/**/*" area/kubernetes: - "kubernetes/**/*" -area/terraform: - - "terraform/**/*" area/templates: - "tmpl/**/*" diff --git a/.github/labels.yaml b/.github/labels.yaml index def5eb19da2..859905109bc 100644 --- a/.github/labels.yaml +++ b/.github/labels.yaml @@ -16,10 +16,6 @@ color: "72ccf3" description: >- Changes made in the tmpl directory -- name: area/terraform - color: "72ccf3" - description: >- - Changes made in the terraform directory # Renovate - name: renovate/ansible color: "ffc300" @@ -31,8 +27,6 @@ color: "ffc300" - name: renovate/helm color: "ffc300" -- name: renovate/terraform - color: "ffc300" # Semantic Type - name: type/patch color: "FFEC19" diff --git a/.github/release-drafter.yaml b/.github/release-drafter.yaml index 3741c28b6f9..0a235cd0541 100644 --- a/.github/release-drafter.yaml +++ b/.github/release-drafter.yaml @@ -12,8 +12,6 @@ categories: labels: ["area/github"] - title: "Ansible" labels: ["area/ansible"] - - title: "Terraform" - labels: ["area/terraform"] - title: "Maintenance" labels: ["docs"] version-resolver: diff --git a/.github/renovate/labels.json5 b/.github/renovate/labels.json5 index 4a04479aefa..494472cb6f7 100644 --- a/.github/renovate/labels.json5 +++ b/.github/renovate/labels.json5 @@ -25,10 +25,6 @@ "matchDatasources": ["galaxy", "galaxy-collection"], "addLabels": ["renovate/ansible"] }, - { - "matchDatasources": ["terraform-provider"], - "addLabels": ["renovate/terraform"] - }, { "matchDatasources": ["github-releases", "github-tags"], "addLabels": ["renovate/github-release"] diff --git a/.github/renovate/semanticCommits.json5 b/.github/renovate/semanticCommits.json5 index 62a3d7c0107..614e44b39da 100644 --- a/.github/renovate/semanticCommits.json5 +++ b/.github/renovate/semanticCommits.json5 @@ -35,7 +35,6 @@ "semanticCommitType": "feat", "semanticCommitScope": "helm" }, - { "matchDatasources": ["helm"], "matchUpdateTypes": ["patch"], @@ -53,30 +52,12 @@ "semanticCommitType": "feat", "semanticCommitScope": "ansible" }, - { "matchDatasources": ["galaxy", "galaxy-collection"], "matchUpdateTypes": ["patch"], "semanticCommitType": "fix", "semanticCommitScope": "ansible" }, - { - "matchDatasources": ["terraform-provider"], - "matchUpdateTypes": ["major"], - "commitMessagePrefix": "feat(terraform)!: " - }, - { - "matchDatasources": ["terraform-provider"], - "matchUpdateTypes": ["minor"], - "semanticCommitType": "feat", - "semanticCommitScope": "terraform" - }, - { - "matchDatasources": ["terraform-provider"], - "matchUpdateTypes": ["patch"], - "semanticCommitType": "fix", - "semanticCommitScope": "terraform" - }, { "matchDatasources": ["github-releases", "github-tags"], "matchUpdateTypes": ["major"], diff --git a/.gitignore b/.gitignore index 28309de382d..bb0e5a985b8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,3 @@ kubeconfig *.key # Ansible xanmanning.k3s* -# Terraform -.terraform -.terraform.tfstate* -terraform.tfstate* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0e7c2ef972..ca0b2d7df76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,3 @@ repos: - --config-file - .yamllint.yaml id: yamllint - - repo: https://github.com/gruntwork-io/pre-commit - rev: v0.1.21 - hooks: - - id: terraform-fmt diff --git a/.taskfiles/TerraformTasks.yml b/.taskfiles/TerraformTasks.yml deleted file mode 100644 index f11a1981dac..00000000000 --- a/.taskfiles/TerraformTasks.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -version: "3" - -tasks: - - init: - desc: Initialize terraform dependencies - dir: "{{.TERRAFORM_DIR}}/cloudflare" - cmds: - - terraform init {{.CLI_ARGS}} - - plan: - desc: Show the changes terraform will make - dir: "{{.TERRAFORM_DIR}}/cloudflare" - cmds: - - terraform plan {{.CLI_ARGS}} - - apply: - desc: Apply the changes to Cloudflare - dir: "{{.TERRAFORM_DIR}}/cloudflare" - cmds: - - terraform apply {{.CLI_ARGS}} diff --git a/.vscode/extensions.json b/.vscode/extensions.json index fc5e1a3d660..3c10ab9d265 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,7 +8,6 @@ "signageos.signageos-vscode-sops", "will-stone.in-any-case", "EditorConfig.editorconfig", - "HashiCorp.terraform", "PKief.material-icon-theme", ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index abc72ddffc2..0b5db4ba79c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,7 +4,6 @@ "**/ansible/**/*.yml": "ansible", "**/ansible/**/*.sops.yml": "yaml", "**/ansible/**/inventory/**/*.yml": "yaml", - "**/terraform/**/*.tf": "terraform", "**/kubernetes/**/*.sops.toml": "plaintext" }, "yaml.schemas": { diff --git a/README.md b/README.md index a58c247274a..860aeb9e336 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Template for deploying k3s backed by Flux -Highly opinionated template for deploying a single [k3s](https://k3s.io) cluster with [Ansible](https://www.ansible.com) and [Terraform](https://www.terraform.io) backed by [Flux](https://toolkit.fluxcd.io/) and [SOPS](https://toolkit.fluxcd.io/guides/mozilla-sops/). +This is a sighly opinionated template for deploying a single [k3s](https://k3s.io) cluster with [Ansible](https://www.ansible.com) and [SOPS](https://toolkit.fluxcd.io/guides/mozilla-sops/). Upon completion you will be able to expose web applications you choose to the internet with [Cloudflare Tunnel](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/) -The purpose here is to showcase how you can deploy an entire Kubernetes cluster and show it off to the world using the [GitOps](https://www.weave.works/blog/what-is-gitops-really) tool [Flux](https://toolkit.fluxcd.io/). When completed, your Git repository will be driving the state of your Kubernetes cluster. In addition with the help of the [Ansible](https://github.com/ansible-collections/community.sops), [Terraform](https://github.com/carlpett/terraform-provider-sops) and [Flux](https://toolkit.fluxcd.io/guides/mozilla-sops/) SOPS integrations you'll be able to commit [Age](https://github.com/FiloSottile/age) encrypted secrets to your public repo. +The purpose here is to showcase how you can deploy an entire Kubernetes cluster and show it off to the world using the [GitOps](https://www.weave.works/blog/what-is-gitops-really) tool [Flux](https://toolkit.fluxcd.io/). When completed, your Git repository will be driving the state of your Kubernetes cluster. In addition with the help of the [Ansible](https://github.com/ansible-collections/community.sops), and [Flux](https://toolkit.fluxcd.io/guides/mozilla-sops/) SOPS integrations you'll be able to commit [Age](https://github.com/FiloSottile/age) encrypted secrets to your public repo. ## Overview @@ -31,16 +31,11 @@ The following components will be installed in your [k3s](https://k3s.io/) cluste _Additional applications include [hajimari](https://github.com/toboshii/hajimari), [error-pages](https://github.com/tarampampam/error-pages), [echo-server](https://github.com/Ealenn/Echo-Server), [system-upgrade-controller](https://github.com/rancher/system-upgrade-controller), [reloader](https://github.com/stakater/Reloader), and [kured](https://github.com/weaveworks/kured)_ -For provisioning the following tools will be used: - -- [Ansible](https://www.ansible.com) - Sets up the operating system and installs k3s -- [Terraform](https://www.terraform.io) - Provisions an existing Cloudflare domain and certain DNS records to be used with your Kubernetes cluster - ## 📝 Prerequisites **Note:** _This template has not been tested on cloud providers like AWS EC2, Hetzner, Scaleway etc... Those cloud offerings probably have a better way of provsioning a Kubernetes cluster and it's advisable to use those instead of the Ansible playbooks included here. This repository can still be tweaked for the GitOps/Flux portion if there's a cluster working in one those environments._ -First and foremost some experience in debugging/troubleshooting problems **and a positive attitude is required** ;) +First and foremost some experience in debugging/troubleshooting problems **and a positive attitude is required** :) ### 📚 Reading material @@ -48,10 +43,10 @@ First and foremost some experience in debugging/troubleshooting problems **and a ### 💻 Systems -- One or more nodes with a fresh install of [Fedora Server 37](https://getfedora.org/en/server/download/) or [Ubuntu 22.04 Server](https://ubuntu.com/download/server) (not minimal). +- One or more nodes with a fresh install of [Fedora Server 37+](https://getfedora.org/en/server/download/) or [Ubuntu 22.04 Server](https://ubuntu.com/download/server) (not minimal). - These nodes can be ARM64/AMD64 bare metal or VMs. - An odd number of control plane nodes, greater than or equal to 3 is required if deploying more than one control plane node. -- A [Cloudflare](https://www.cloudflare.com/) account with a domain, this will be managed by Terraform and external-dns. You can [register new domains](https://www.cloudflare.com/products/registrar/) directly thru Cloudflare. +- A [Cloudflare](https://www.cloudflare.com/) account with a domain, this will be managed by external-dns. You can [register new domains](https://www.cloudflare.com/products/registrar/) directly thru Cloudflare. 📍 It is recommended to have 3 master nodes for a highly available control plane. @@ -80,26 +75,27 @@ Clone **your new repo** to you local workstation and `cd` into it. 1. Install the following CLI tools on your workstation, if you are **NOT** using [Homebrew](https://brew.sh/) on MacOS or Linux **ignore** steps 4 and 5. - * Required: [age](https://github.com/FiloSottile/age), [ansible](https://www.ansible.com), [flux](https://toolkit.fluxcd.io/), [weave-gitops](https://docs.gitops.weave.works/docs/installation/weave-gitops/), [go-task](https://github.com/go-task/task), [direnv](https://github.com/direnv/direnv), [ipcalc](http://jodies.de/ipcalc), [jq](https://stedolan.github.io/jq/), [kubectl](https://kubernetes.io/docs/tasks/tools/), [python-pip3](https://pypi.org/project/pip/), [pre-commit](https://github.com/pre-commit/pre-commit), [sops v3](https://github.com/mozilla/sops), [terraform](https://www.terraform.io), [yq v4](https://github.com/mikefarah/yq) + - Required: [age](https://github.com/FiloSottile/age), [ansible](https://www.ansible.com), [flux](https://toolkit.fluxcd.io/), [weave-gitops](https://docs.gitops.weave.works/docs/installation/weave-gitops/), [go-task](https://github.com/go-task/task), [direnv](https://github.com/direnv/direnv), [ipcalc](http://jodies.de/ipcalc), [jq](https://stedolan.github.io/jq/), [kubectl](https://kubernetes.io/docs/tasks/tools/), [python-pip3](https://pypi.org/project/pip/), [pre-commit](https://github.com/pre-commit/pre-commit), [sops v3](https://github.com/mozilla/sops), [terraform](https://www.terraform.io), [yq v4](https://github.com/mikefarah/yq) - * Recommended: [helm](https://helm.sh/), [kustomize](https://github.com/kubernetes-sigs/kustomize), [stern](https://github.com/stern/stern), [yamllint](https://github.com/adrienverge/yamllint) + - Recommended: [helm](https://helm.sh/), [kustomize](https://github.com/kubernetes-sigs/kustomize), [stern](https://github.com/stern/stern), [yamllint](https://github.com/adrienverge/yamllint) 2. This guide heavily relies on [go-task](https://github.com/go-task/task) as a framework for setting things up. It is advised to learn and understand the commands it is running under the hood. 3. Install Python 3 and pip3 using your Linux OS package manager, or Homebrew if using MacOS. - - Ensure `pip3` is working on your command line by running `pip3 --version` + + - Ensure `pip3` is working on your command line by running `pip3 --version` 4. [Homebrew] Install [go-task](https://github.com/go-task/task) - ```sh - brew install go-task/tap/go-task - ``` + ```sh + brew install go-task/tap/go-task + ``` 5. [Homebrew] Install workstation dependencies - ```sh - task init - ``` + ```sh + task init + ``` ### ⚠️ pre-commit @@ -107,45 +103,45 @@ It is advisable to install [pre-commit](https://pre-commit.com/) and the pre-com 1. Enable Pre-Commit - ```sh - task precommit:init - ``` + ```sh + task precommit:init + ``` 2. Update Pre-Commit, though it will occasionally make mistakes, so verify its results. - ```sh - task precommit:update - ``` + ```sh + task precommit:update + ``` ### 🔐 Setting up Age -📍 Here we will create a Age Private and Public key. Using [SOPS](https://github.com/mozilla/sops) with [Age](https://github.com/FiloSottile/age) allows us to encrypt secrets and use them in Ansible, Terraform and Flux. +📍 Here we will create a Age Private and Public key. Using [SOPS](https://github.com/mozilla/sops) with [Age](https://github.com/FiloSottile/age) allows us to encrypt secrets and use them in Ansible and Flux. 1. Create a Age Private / Public Key - ```sh - age-keygen -o age.agekey - ``` + ```sh + age-keygen -o age.agekey + ``` 2. Set up the directory for the Age key and move the Age file to it - ```sh - mkdir -p ~/.config/sops/age - mv age.agekey ~/.config/sops/age/keys.txt - ``` + ```sh + mkdir -p ~/.config/sops/age + mv age.agekey ~/.config/sops/age/keys.txt + ``` 3. Export the `SOPS_AGE_KEY_FILE` variable in your `bashrc`, `zshrc` or `config.fish` and source it, e.g. - ```sh - export SOPS_AGE_KEY_FILE=~/.config/sops/age/keys.txt - source ~/.bashrc - ``` + ```sh + export SOPS_AGE_KEY_FILE=~/.config/sops/age/keys.txt + source ~/.bashrc + ``` 4. Fill out the Age public key in the appropriate variable in configuration section below, **note** the public key should start with `age`... -### ☁️ Global Cloudflare API Key +### ☁️ Cloudflare API Key -In order to use Terraform and `cert-manager` with the Cloudflare DNS challenge you will need to create a API key. +In order to use `cert-manager` with the Cloudflare DNS challenge you will need to create a API key. 1. Head over to Cloudflare and create a API key by going [here](https://dash.cloudflare.com/profile/api-tokens). @@ -155,29 +151,47 @@ In order to use Terraform and `cert-manager` with the Cloudflare DNS challenge y 📍 You may wish to update this later on to a Cloudflare **API Token** which can be scoped to certain resources. I do not recommend using a Cloudflare **API Key**, however for the purposes of this template it is easier getting started without having to define which scopes and resources are needed. For more information see the [Cloudflare docs on API Keys and Tokens](https://developers.cloudflare.com/api/). +### ☁️ Cloudflare Tunnel + +In order to expose services to the internet you will need to create a [Cloudflare Tunnel](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/). + +1. Authenticate cloudflared to your domain + + ```sh + cloudflared tunnel login + ``` + +2. Create the tunnel + + ```sh + cloudflared tunnel create k8s + ``` + +3. In the `~/.cloudflared` directory there will be a json file with details you need to populate in configuration section below. You can ignore the `cert.pem` file. + ### 📄 Configuration -📍 The `.config.env` file contains necessary configuration that is needed by Ansible, Terraform and Flux. +📍 The `.config.env` file contains necessary configuration that is needed by Ansible and Flux. 1. Copy the `.config.sample.env` to `.config.env` and start filling out all the environment variables. - **All are required** unless otherwise noted in the comments. + **All are required** unless otherwise noted in the comments. - ```sh - cp .config.sample.env .config.env - ``` + ```sh + cp .config.sample.env .config.env + ``` 2. Once that is done, verify the configuration is correct by running: - ```sh - task verify - ``` + ```sh + task verify + ``` 3. If you do not encounter any errors run start having the script wire up the templated files and place them where they need to be. - ```sh - task configure - ``` + ```sh + task configure + ``` ### ⚡ Preparing Fedora or Ubuntu Server with Ansible @@ -191,33 +205,33 @@ In order to use Terraform and `cert-manager` with the Cloudflare DNS challenge y 2. Install the Ansible deps - ```sh - task ansible:init - ``` + ```sh + task ansible:init + ``` 3. Verify Ansible can view your config - ```sh - task ansible:list - ``` + ```sh + task ansible:list + ``` 4. Verify Ansible can ping your nodes - ```sh - task ansible:ping - ``` + ```sh + task ansible:ping + ``` 5. Run the Fedora/Ubuntu Server Ansible prepare playbook - ```sh - task ansible:prepare - ``` + ```sh + task ansible:prepare + ``` 6. Reboot the nodes (if not done in step 5) - ```sh - task ansible:force-reboot - ``` + ```sh + task ansible:force-reboot + ``` ### ⛵ Installing k3s with Ansible @@ -227,58 +241,30 @@ In order to use Terraform and `cert-manager` with the Cloudflare DNS challenge y 1. Verify Ansible can view your config - ```sh - task ansible:list - ``` + ```sh + task ansible:list + ``` 2. Verify Ansible can ping your nodes - ```sh - task ansible:ping - ``` + ```sh + task ansible:ping + ``` 3. Install k3s with Ansible - ```sh - task ansible:install - ``` + ```sh + task ansible:install + ``` 4. Verify the nodes are online - ```sh - task cluster:nodes - # NAME STATUS ROLES AGE VERSION - # k8s-0 Ready control-plane,master 4d20h v1.21.5+k3s1 - # k8s-1 Ready worker 4d20h v1.21.5+k3s1 - ``` - -### ☁️ Configuring Cloudflare DNS with Terraform - -📍 Review the Terraform scripts under `./terraform/cloudflare/` and make sure you understand what it's doing (no really review it). - -If your domain already has existing DNS records **be sure to export those DNS settings before you continue**. - -1. Pull in the Terraform deps - - ```sh - task terraform:init - ``` - -2. Review the changes Terraform will make to your Cloudflare domain - - ```sh - task terraform:plan - ``` - -3. Have Terraform apply your Cloudflare settings - - ```sh - task terraform:apply - ``` - -If Terraform was ran successfully you can log into Cloudflare and validate the DNS records are present. - -The cluster application [external-dns](https://github.com/kubernetes-sigs/external-dns) will be managing the rest of the DNS records you will need. + ```sh + task cluster:nodes + # NAME STATUS ROLES AGE VERSION + # k8s-0 Ready control-plane,master 4d20h v1.21.5+k3s1 + # k8s-1 Ready worker 4d20h v1.21.5+k3s1 + ``` ### 🔹 GitOps with Flux @@ -286,42 +272,42 @@ The cluster application [external-dns](https://github.com/kubernetes-sigs/extern 1. Verify Flux can be installed - ```sh - task cluster:verify - # ► checking prerequisites - # ✔ kubectl 1.21.5 >=1.18.0-0 - # ✔ Kubernetes 1.21.5+k3s1 >=1.16.0-0 - # ✔ prerequisites checks passed - ``` + ```sh + task cluster:verify + # ► checking prerequisites + # ✔ kubectl 1.21.5 >=1.18.0-0 + # ✔ Kubernetes 1.21.5+k3s1 >=1.16.0-0 + # ✔ prerequisites checks passed + ``` 2. Push you changes to git - 📍 **Verify** all the `*.sops.yaml` and `*.sops.yml` files under the `./ansible`, `./kubernetes`, and `./terraform` folders are **encrypted** with SOPS + 📍 **Verify** all the `*.sops.yaml` and `*.sops.yml` files under the `./ansible`, `./kubernetes`, and `./terraform` folders are **encrypted** with SOPS - ```sh - git add -A - git commit -m "Initial commit :rocket:" - git push - ``` + ```sh + git add -A + git commit -m "Initial commit :rocket:" + git push + ``` 3. Install Flux and sync the cluster to the Git repository - ```sh - task cluster:install - # namespace/flux-system configured - # customresourcedefinition.apiextensions.k8s.io/alerts.notification.toolkit.fluxcd.io created - ``` + ```sh + task cluster:install + # namespace/flux-system configured + # customresourcedefinition.apiextensions.k8s.io/alerts.notification.toolkit.fluxcd.io created + ``` 4. Verify Flux components are running in the cluster - ```sh - task cluster:pods -- -n flux-system - # NAME READY STATUS RESTARTS AGE - # helm-controller-5bbd94c75-89sb4 1/1 Running 0 1h - # kustomize-controller-7b67b6b77d-nqc67 1/1 Running 0 1h - # notification-controller-7c46575844-k4bvr 1/1 Running 0 1h - # source-controller-7d6875bcb4-zqw9f 1/1 Running 0 1h - ``` + ```sh + task cluster:pods -- -n flux-system + # NAME READY STATUS RESTARTS AGE + # helm-controller-5bbd94c75-89sb4 1/1 Running 0 1h + # kustomize-controller-7b67b6b77d-nqc67 1/1 Running 0 1h + # notification-controller-7c46575844-k4bvr 1/1 Running 0 1h + # source-controller-7d6875bcb4-zqw9f 1/1 Running 0 1h + ``` ### 🎤 Verification Steps @@ -335,45 +321,45 @@ task cluster:resources 1. View the Flux Git Repositories - ```sh - task cluster:gitrepositories - ``` + ```sh + task cluster:gitrepositories + ``` 2. View the Flux kustomizations - ```sh - task cluster:kustomizations - ``` + ```sh + task cluster:kustomizations + ``` 3. View all the Flux Helm Releases - ```sh - task cluster:helmreleases - ``` + ```sh + task cluster:helmreleases + ``` 4. View all the Flux Helm Repositories - ```sh - task cluster:helmrepositories - ``` + ```sh + task cluster:helmrepositories + ``` 5. View all the Pods - ```sh - task cluster:pods - ``` + ```sh + task cluster:pods + ``` 6. View all the certificates and certificate requests - ```sh - task cluster:certificates - ``` + ```sh + task cluster:certificates + ``` 7. View all the ingresses - ```sh - task cluster:ingresses - ``` + ```sh + task cluster:ingresses + ``` 🏆 **Congratulations** if all goes smooth you'll have a Kubernetes cluster managed by Flux, your Git repository is driving the state of your cluster. @@ -389,15 +375,13 @@ task cluster:resources ### 🌐 DNS -📍 The [external-dns](https://github.com/kubernetes-sigs/external-dns) application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only public domain exposed on your Cloudflare domain. In order to make additional applications public you must set an ingress annotation (`external-dns.alpha.kubernetes.io/target`) like done in the `HelmRelease` for `echo-server`. You do not need to use Terraform to create additional DNS records unless you need a record outside the purposes of your Kubernetes cluster (e.g. setting up MX records). +📍 The [external-dns](https://github.com/kubernetes-sigs/external-dns) application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only public domain exposed on your Cloudflare domain. In order to make additional applications public you must set an ingress annotation (`external-dns.alpha.kubernetes.io/target`) like done in the `HelmRelease` for `echo-server`. [k8s_gateway](https://github.com/ori-edge/k8s_gateway) is deployed on the IP choosen for `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}`. Inorder to test DNS you can point your clients DNS to the `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}` IP address and load `https://hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` in your browser. You can also try debugging with the command `dig`, e.g. `dig @${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR} hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` and you should get a valid answer containing your `${BOOTSTRAP_METALLB_INGRESS_ADDR}` IP address. -If your router (or Pi-Hole, Adguard Home or whatever) supports conditional DNS forwarding (also know as split-horizon DNS) you may have DNS requests for `${SECRET_DOMAIN}` only point to the `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}` IP address. This will ensure only DNS requests for `${SECRET_DOMAIN}` will only get routed to your [k8s_gateway](https://github.com/ori-edge/k8s_gateway) service thus providing DNS resolution to your cluster applications/ingresses. - -To access services from the outside world port forwarded `80` and `443` in your router to the `${BOOTSTRAP_METALLB_INGRESS_ADDR}` IP, in a few moments head over to your browser and you _should_ be able to access `https://echo-server.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` from a device outside your LAN. +If your router (or Pi-Hole, Adguard Home or whatever) supports conditional DNS forwarding (also know as split-horizon DNS) you may have DNS requests for `${SECRET_DOMAIN}` only point to the `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}` IP address. This will ensure only DNS requests for `${SECRET_DOMAIN}` will only get routed to your [k8s_gateway](https://github.com/ori-edge/k8s_gateway) service thus providing DNS resolution to your cluster applications/ingresses. Now if nothing is working, that is expected. This is DNS after all! @@ -415,25 +399,25 @@ Flux is pull-based by design meaning it will periodically check your git reposit 1. Webhook URL - Your webhook receiver will be deployed on `https://flux-webhook.${BOOTSTRAP_CLOUDFLARE_DOMAIN}/hook/:hookId`. In order to find out your hook id you can run the following command: - ```sh - kubectl -n flux-system get receiver/github-receiver - # NAME AGE READY STATUS - # github-receiver 6h8m True Receiver initialized with URL: /hook/12ebd1e363c641dc3c2e430ecf3cee2b3c7a5ac9e1234506f6f5f3ce1230e123 - ``` + ```sh + kubectl -n flux-system get receiver/github-receiver + # NAME AGE READY STATUS + # github-receiver 6h8m True Receiver initialized with URL: /hook/12ebd1e363c641dc3c2e430ecf3cee2b3c7a5ac9e1234506f6f5f3ce1230e123 + ``` - So if my domain was `onedr0p.com` the full url would look like this: + So if my domain was `onedr0p.com` the full url would look like this: - ```text - https://flux-webhook.onedr0p.com/hook/12ebd1e363c641dc3c2e430ecf3cee2b3c7a5ac9e1234506f6f5f3ce1230e123 - ``` + ```text + https://flux-webhook.onedr0p.com/hook/12ebd1e363c641dc3c2e430ecf3cee2b3c7a5ac9e1234506f6f5f3ce1230e123 + ``` 2. Webhook secret - Your webhook secret can be found by decrypting the `secret.sops.yaml` using the following command: - ```sh - sops -d ./kubernetes/apps/flux-system/addons/webhooks/github/secret.sops.yaml | yq .stringData.token - ``` + ```sh + sops -d ./kubernetes/apps/flux-system/addons/webhooks/github/secret.sops.yaml | yq .stringData.token + ``` - **Note:** Don't forget to update the `BOOTSTRAP_FLUX_GITHUB_WEBHOOK_SECRET` variable in your `.config.env` file so it matches the generated secret if applicable + **Note:** Don't forget to update the `BOOTSTRAP_FLUX_GITHUB_WEBHOOK_SECRET` variable in your `.config.env` file so it matches the generated secret if applicable Now that you have the webhook url and secret, it's time to set everything up on the Github repository side. Navigate to the settings of your repository on Github, under "Settings/Webhooks" press the "Add webhook" button. Fill in the webhook url and your secret. @@ -457,70 +441,70 @@ By default this template only works on a public GitHub repository, it is advised The benefits of a public repository include: -* Debugging or asking for help, you can provide a link to a resource you are having issues with. -* Adding a topic to your repository of `k8s-at-home` to be included in the [k8s-at-home-search](https://whazor.github.io/k8s-at-home-search/). This search helps people discover different configurations of Helm charts across others Flux based repositories. +- Debugging or asking for help, you can provide a link to a resource you are having issues with. +- Adding a topic to your repository of `k8s-at-home` to be included in the [k8s-at-home-search](https://whazor.github.io/k8s-at-home-search/). This search helps people discover different configurations of Helm charts across others Flux based repositories.
Expand to read guide on adding Flux SSH authentication - 1. Generate new SSH key: - ```sh - ssh-keygen -t ecdsa -b 521 -C "github-deploy-key" -f ./kubernetes/bootstrap/github-deploy.key -q -P "" - ``` - 2. Paste public key in the deploy keys section of your repository settings - 3. Create sops secret in `./kubernetes/bootstrap/github-deploy-key.sops.yaml` with the contents of: - ```yaml - apiVersion: v1 - kind: Secret - metadata: - name: github-deploy-key - namespace: flux-system - stringData: - # 3a. Contents of github-deploy-key - identity: | - -----BEGIN OPENSSH PRIVATE KEY----- - ... - -----END OPENSSH PRIVATE KEY----- - # 3b. Output of curl --silent https://api.github.com/meta | jq --raw-output '"github.com "+.ssh_keys[]' - known_hosts: | - github.com ssh-ed25519 ... - github.com ecdsa-sha2-nistp256 ... - github.com ssh-rsa ... - ``` - 4. Encrypt secret: - ```sh - sops --encrypt --in-place ./kubernetes/bootstrap/github-deploy-key.sops.yaml - ``` - 5. Apply secret to cluster: - ```sh - sops --decrypt ./kubernetes/bootstrap/github-deploy-key.sops.yaml | kubectl apply -f - - ``` - 6. Update `./kubernetes/flux/config/cluster.yaml`: - ```yaml - apiVersion: source.toolkit.fluxcd.io/v1beta2 - kind: GitRepository - metadata: - name: home-kubernetes - namespace: flux-system - spec: - interval: 10m - # 6a: Change this to your user and repo names - url: ssh://git@github.com/$user/$repo - ref: - branch: main - secretRef: - name: github-deploy-key - ``` - 7. Commit and push changes - 8. Force flux to reconcile your changes - ```sh - task cluster:reconcile - ``` - 9. Verify git repository is now using SSH: - ```sh - task cluster:gitrepositories - ``` - 10. Optionally set your repository to Private in your repository settings. +1. Generate new SSH key: + ```sh + ssh-keygen -t ecdsa -b 521 -C "github-deploy-key" -f ./kubernetes/bootstrap/github-deploy.key -q -P "" + ``` +2. Paste public key in the deploy keys section of your repository settings +3. Create sops secret in `./kubernetes/bootstrap/github-deploy-key.sops.yaml` with the contents of: + ```yaml + apiVersion: v1 + kind: Secret + metadata: + name: github-deploy-key + namespace: flux-system + stringData: + # 3a. Contents of github-deploy-key + identity: | + -----BEGIN OPENSSH PRIVATE KEY----- + ... + -----END OPENSSH PRIVATE KEY----- + # 3b. Output of curl --silent https://api.github.com/meta | jq --raw-output '"github.com "+.ssh_keys[]' + known_hosts: | + github.com ssh-ed25519 ... + github.com ecdsa-sha2-nistp256 ... + github.com ssh-rsa ... + ``` +4. Encrypt secret: + ```sh + sops --encrypt --in-place ./kubernetes/bootstrap/github-deploy-key.sops.yaml + ``` +5. Apply secret to cluster: + ```sh + sops --decrypt ./kubernetes/bootstrap/github-deploy-key.sops.yaml | kubectl apply -f - + ``` +6. Update `./kubernetes/flux/config/cluster.yaml`: + ```yaml + apiVersion: source.toolkit.fluxcd.io/v1beta2 + kind: GitRepository + metadata: + name: home-kubernetes + namespace: flux-system + spec: + interval: 10m + # 6a: Change this to your user and repo names + url: ssh://git@github.com/$user/$repo + ref: + branch: main + secretRef: + name: github-deploy-key + ``` +7. Commit and push changes +8. Force flux to reconcile your changes + ```sh + task cluster:reconcile + ``` +9. Verify git repository is now using SSH: + ```sh + task cluster:gitrepositories + ``` +10. Optionally set your repository to Private in your repository settings.
### 💨 Kubernetes Dashboard @@ -538,13 +522,20 @@ You should be able to access the dashboard at `https://kubernetes.${SECRET_DOMAI Below is a general guide on trying to debug an issue with an resource or application. For example, if a workload/resource is not showing up or a pod has started but in a `CrashLoopBackOff` or `Pending` state. 1. Start by checking all Flux Kustomizations and verify they are healthy. - - `flux get ks -A` + +- `flux get ks -A` + 2. Then check all the Flux Helm Releases and verify they are healthy. - - `flux get hr -A` + +- `flux get hr -A` + 3. Then check the if the pod is present. - - `kubectl -n get pods` + +- `kubectl -n get pods` + 4. Then check the logs of the pod if its there. - - `kubectl -n logs -f` + +- `kubectl -n logs -f` Note: If a resource exists, running `kubectl -n describe ` might give you insight into what the problem(s) could be. diff --git a/Taskfile.yml b/Taskfile.yml index 8b7dec298e4..7401b08f867 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -4,7 +4,6 @@ version: "3" vars: KUBERNETES_DIR: "{{.ROOT_DIR}}/kubernetes" ANSIBLE_DIR: "{{.ROOT_DIR}}/ansible" - TERRAFORM_DIR: "{{.ROOT_DIR}}/terraform" dotenv: [".config.env"] @@ -16,10 +15,8 @@ includes: ansible: .taskfiles/AnsibleTasks.yml cluster: .taskfiles/ClusterTasks.yml precommit: .taskfiles/PrecommitTasks.yml - terraform: .taskfiles/TerraformTasks.yml tasks: - init: desc: Initialize workstation dependencies with Brew cmds: @@ -33,6 +30,7 @@ tasks: DEPS: >- age ansible + cloudflare/cloudflare/cloudflared direnv fluxcd/tap/flux go-task/tap/go-task @@ -45,8 +43,6 @@ tasks: prettier sops stern - terraform - tflint weaveworks/tap/gitops yamllint yq diff --git a/configure b/configure index 316c46cf2c2..6e655f2875f 100755 --- a/configure +++ b/configure @@ -51,15 +51,9 @@ main() { envsubst < "${PROJECT_DIR}/tmpl/kubernetes/cert-manager-secret.sops.yaml" \ > "${PROJECT_DIR}/kubernetes/apps/cert-manager/cert-manager/issuers/secret.sops.yaml" sops --encrypt --in-place "${PROJECT_DIR}/kubernetes/apps/cert-manager/cert-manager/issuers/secret.sops.yaml" - envsubst < "${PROJECT_DIR}/tmpl/kubernetes/cloudflare-ddns-secret.sops.yaml" \ - > "${PROJECT_DIR}/kubernetes/apps/networking/cloudflare-ddns/app/secret.sops.yaml" - sops --encrypt --in-place "${PROJECT_DIR}/kubernetes/apps/networking/cloudflare-ddns/app/secret.sops.yaml" - envsubst < "${PROJECT_DIR}/tmpl/kubernetes/external-dns-secret.sops.yaml" \ - > "${PROJECT_DIR}/kubernetes/apps/networking/external-dns/app/secret.sops.yaml" - sops --encrypt --in-place "${PROJECT_DIR}/kubernetes/apps/networking/external-dns/app/secret.sops.yaml" - envsubst < "${PROJECT_DIR}/tmpl/terraform/secret.sops.yaml" \ - > "${PROJECT_DIR}/terraform/cloudflare/secret.sops.yaml" - sops --encrypt --in-place "${PROJECT_DIR}/terraform/cloudflare/secret.sops.yaml" + envsubst < "${PROJECT_DIR}/tmpl/kubernetes/cloudflared-secret.sops.yaml" \ + > "${PROJECT_DIR}/kubernetes/apps/networking/cloudflared/app/secret.sops.yaml" + sops --encrypt --in-place "${PROJECT_DIR}/kubernetes/apps/networking/cloudflared/app/secret.sops.yaml" # generate ansible settings envsubst < "${PROJECT_DIR}/tmpl/ansible/kube-vip.yml" \ > "${PROJECT_DIR}/ansible/inventory/group_vars/kubernetes/kube-vip.yml" @@ -229,7 +223,6 @@ verify_binaries() { _has_binary "sops" _has_binary "ssh" _has_binary "task" - _has_binary "terraform" _has_binary "yq" if ! [[ "$(sops --version)" =~ 3\.[0-9]+\.[0-9]+ ]]; then _log "ERROR(${FUNCNAME[0]})" "Incompatible sops version, make sure you are using the latest release of github.com/mozilla/sops" @@ -287,8 +280,8 @@ verify_master_count() { } verify_cloudflare() { - local account_zone= local errors= + local account_zone= _has_envar "BOOTSTRAP_CLOUDFLARE_APIKEY" "true" _has_envar "BOOTSTRAP_CLOUDFLARE_DOMAIN" "true" _has_envar "BOOTSTRAP_CLOUDFLARE_EMAIL" "true" @@ -305,6 +298,23 @@ verify_cloudflare() { _log "ERROR(${FUNCNAME[0]})" "Unable to get Cloudflare Account and Zone information ${errors}" exit 1 fi + local cloudflare_tunnel= + _has_envar "BOOTSTRAP_CLOUDFLARE_ACCOUNT_TAG" "true" + _has_envar "BOOTSTRAP_CLOUDFLARE_TUNNEL_SECRET" "true" + _has_envar "BOOTSTRAP_CLOUDFLARE_TUNNEL_ID" "true" + # Try to retrieve the Cloudflare Tunnel information from Cloudflare's API + cloudflare_tunnel=$(curl -s -X GET "https://api.cloudflare.com/client/v4/accounts/${BOOTSTRAP_CLOUDFLARE_ACCOUNT_TAG}/cfd_tunnel/${BOOTSTRAP_CLOUDFLARE_TUNNEL_ID}" \ + -H "X-Auth-Email: ${BOOTSTRAP_CLOUDFLARE_EMAIL}" \ + -H "X-Auth-Key: ${BOOTSTRAP_CLOUDFLARE_APIKEY}" \ + -H "Content-Type: application/json" + ) + if [[ "$(echo "${cloudflare_tunnel}" | jq ".success")" == "true" ]]; then + _log "INFO(${FUNCNAME[0]})" "Verified Cloudflare Tunnel information" + else + errors=$(echo "${cloudflare_tunnel}" | jq -c ".errors") + _log "ERROR(${FUNCNAME[0]})" "Unable to get Cloudflare Tunnel information ${errors}" + exit 1 + fi } verify_ansible_hosts() { diff --git a/kubernetes/apps/networking/cloudflare-ddns/app/cloudflare-ddns.sh b/kubernetes/apps/networking/cloudflare-ddns/app/cloudflare-ddns.sh deleted file mode 100755 index 0ba5b7039a4..00000000000 --- a/kubernetes/apps/networking/cloudflare-ddns/app/cloudflare-ddns.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit - -current_ipv4="$(curl -s https://ipv4.icanhazip.com/)" -zone_id=$(curl -s -X GET \ - "https://api.cloudflare.com/client/v4/zones?name=${CLOUDFLARE_RECORD_NAME#*.}&status=active" \ - -H "X-Auth-Email: ${CLOUDFLARE_EMAIL}" \ - -H "X-Auth-Key: ${CLOUDFLARE_APIKEY}" \ - -H "Content-Type: application/json" \ - | jq --raw-output ".result[0] | .id" -) -record_ipv4=$(curl -s -X GET \ - "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?name=${CLOUDFLARE_RECORD_NAME}&type=A" \ - -H "X-Auth-Email: ${CLOUDFLARE_EMAIL}" \ - -H "X-Auth-Key: ${CLOUDFLARE_APIKEY}" \ - -H "Content-Type: application/json" \ -) -old_ip4=$(echo "$record_ipv4" | jq --raw-output '.result[0] | .content') -if [[ "${current_ipv4}" == "${old_ip4}" ]]; then - printf "%s - IP Address '%s' has not changed" "$(date -u)" "${current_ipv4}" - exit 0 -fi -record_ipv4_identifier="$(echo "$record_ipv4" | jq --raw-output '.result[0] | .id')" -update_ipv4=$(curl -s -X PUT \ - "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_ipv4_identifier}" \ - -H "X-Auth-Email: ${CLOUDFLARE_EMAIL}" \ - -H "X-Auth-Key: ${CLOUDFLARE_APIKEY}" \ - -H "Content-Type: application/json" \ - --data "{\"id\":\"${zone_id}\",\"type\":\"A\",\"proxied\":true,\"name\":\"${CLOUDFLARE_RECORD_NAME}\",\"content\":\"${current_ipv4}\"}" \ -) -if [[ "$(echo "$update_ipv4" | jq --raw-output '.success')" == "true" ]]; then - printf "%s - Success - IP Address '%s' has been updated" "$(date -u)" "${current_ipv4}" - exit 0 -else - printf "%s - Yikes - Updating IP Address '%s' has failed" "$(date -u)" "${current_ipv4}" - exit 1 -fi diff --git a/kubernetes/apps/networking/cloudflare-ddns/app/helmrelease.yaml b/kubernetes/apps/networking/cloudflare-ddns/app/helmrelease.yaml deleted file mode 100644 index ce727f5af16..00000000000 --- a/kubernetes/apps/networking/cloudflare-ddns/app/helmrelease.yaml +++ /dev/null @@ -1,53 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: cloudflare-ddns - namespace: networking -spec: - interval: 15m - chart: - spec: - chart: app-template - version: 1.4.0 - sourceRef: - kind: HelmRepository - name: bjw-s - namespace: flux-system - maxHistory: 3 - install: - createNamespace: true - remediation: - retries: 3 - upgrade: - cleanupOnFail: true - remediation: - retries: 3 - uninstall: - keepHistory: false - values: - controller: - type: cronjob - cronjob: - concurrencyPolicy: Forbid - schedule: "@hourly" - restartPolicy: OnFailure - image: - repository: ghcr.io/onedr0p/kubernetes-kubectl - tag: 1.27.1@sha256:2067b52145cdcb99b1db4e92fa114babc0a8a91e08711cbe1aae05ba5a277dd9 - command: ["/bin/bash", "/app/cloudflare-ddns.sh"] - envFrom: - - secretRef: - name: cloudflare-ddns-secret - service: - main: - enabled: false - persistence: - config: - enabled: true - type: configMap - name: cloudflare-ddns-configmap - subPath: cloudflare-ddns.sh - mountPath: /app/cloudflare-ddns.sh - defaultMode: 0775 - readOnly: true diff --git a/kubernetes/apps/networking/cloudflared/app/configs/config.yaml b/kubernetes/apps/networking/cloudflared/app/configs/config.yaml new file mode 100644 index 00000000000..5603c2f4db1 --- /dev/null +++ b/kubernetes/apps/networking/cloudflared/app/configs/config.yaml @@ -0,0 +1,18 @@ +--- +# The `ingress` block tells cloudflared which local service to route incoming +# requests to. For more about ingress rules, see +# https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ingress +# +# Remember, these rules route traffic from cloudflared to a local service. To route traffic +# from the internet to cloudflared, run `cloudflared tunnel route dns `. +# E.g. `cloudflared tunnel route dns example-tunnel tunnel.example.com`. +ingress: + - hostname: "${SECRET_DOMAIN}" + service: https://ingress-nginx-controller.networking.svc.cluster.local:443 + originRequest: + originServerName: "ipv4.${SECRET_DOMAIN}" + - hostname: "*.${SECRET_DOMAIN}" + service: https://ingress-nginx-controller.networking.svc.cluster.local:443 + originRequest: + originServerName: "ipv4.${SECRET_DOMAIN}" + - service: http_status:404 diff --git a/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml b/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml new file mode 100644 index 00000000000..6ea800c5ab4 --- /dev/null +++ b/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: externaldns.k8s.io/v1alpha1 +kind: DNSEndpoint +metadata: + name: cloudflared + namespace: networking + annotations: + external-dns.alpha.kubernetes.io/target: "ipv4.${SECRET_DOAMIN}" +spec: + endpoints: + - dnsName: "ipv4.${SECRET_DOAMIN}" + recordType: CNAME + targets: ["${SECRET_CLOUDFLARE_TUNNEL_ID}.cfargotunnel.com"] diff --git a/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml b/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml new file mode 100644 index 00000000000..c39fc1c5346 --- /dev/null +++ b/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml @@ -0,0 +1,102 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: &app cloudflared + namespace: networking +spec: + interval: 15m + chart: + spec: + chart: app-template + version: 1.4.0 + sourceRef: + kind: HelmRepository + name: bjw-s-charts + namespace: flux-system + maxHistory: 3 + install: + createNamespace: true + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + controller: + replicas: 1 + strategy: RollingUpdate + annotations: + reloader.stakater.com/auto: "true" + image: + repository: docker.io/cloudflare/cloudflared + tag: 2023.5.0 + env: + NO_AUTOUPDATE: "true" + TUNNEL_CRED_FILE: /etc/cloudflared/creds/credentials.json + TUNNEL_METRICS: 0.0.0.0:8080 + TUNNEL_TRANSPORT_PROTOCOL: auto + TUNNEL_ID: + valueFrom: + secretKeyRef: + name: cloudflared-secret + key: TUNNEL_ID + args: + - tunnel + - --config + - /etc/cloudflared/config/config.yaml + - run + - "$(TUNNEL_ID)" + service: + main: + ports: + http: + port: 8080 + serviceMonitor: + main: + enabled: true + endpoints: + - port: http + scheme: http + path: /metrics + interval: 1m + scrapeTimeout: 30s + probes: + liveness: &probes + enabled: true + custom: true + spec: + httpGet: + path: /ready + port: http + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + readiness: *probes + startup: + enabled: false + persistence: + config: + enabled: true + type: configMap + name: cloudflared-configmap + subPath: config.yaml + mountPath: /etc/cloudflared/config/config.yaml + readOnly: true + creds: + enabled: true + type: secret + name: cloudflared-secret + subPath: credentials.json + mountPath: /etc/cloudflared/creds/credentials.json + readOnly: true + resources: + requests: + cpu: 5m + memory: 10Mi + limits: + memory: 256Mi diff --git a/kubernetes/apps/networking/cloudflare-ddns/app/kustomization.yaml b/kubernetes/apps/networking/cloudflared/app/kustomization.yaml similarity index 62% rename from kubernetes/apps/networking/cloudflare-ddns/app/kustomization.yaml rename to kubernetes/apps/networking/cloudflared/app/kustomization.yaml index b726e78c2ea..0536740db62 100644 --- a/kubernetes/apps/networking/cloudflare-ddns/app/kustomization.yaml +++ b/kubernetes/apps/networking/cloudflared/app/kustomization.yaml @@ -3,13 +3,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: networking resources: + - ./dnsendpoint.yaml - ./secret.sops.yaml - ./helmrelease.yaml configMapGenerator: - - name: cloudflare-ddns-configmap + - name: cloudflared-configmap files: - - ./cloudflare-ddns.sh + - ./configs/config.yaml generatorOptions: disableNameSuffixHash: true - annotations: - kustomize.toolkit.fluxcd.io/substitute: disabled diff --git a/kubernetes/apps/networking/cloudflare-ddns/ks.yaml b/kubernetes/apps/networking/cloudflared/ks.yaml similarity index 61% rename from kubernetes/apps/networking/cloudflare-ddns/ks.yaml rename to kubernetes/apps/networking/cloudflared/ks.yaml index 3361b4ea471..898b681a725 100644 --- a/kubernetes/apps/networking/cloudflare-ddns/ks.yaml +++ b/kubernetes/apps/networking/cloudflared/ks.yaml @@ -2,10 +2,12 @@ apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: cluster-apps-cloudflare-ddns + name: cluster-apps-cloudflared namespace: flux-system spec: - path: ./kubernetes/apps/networking/cloudflare-ddns/app + dependsOn: + - name: cluster-apps-ingress-nginx + path: ./kubernetes/apps/networking/cloudflared/app prune: true sourceRef: kind: GitRepository @@ -13,7 +15,8 @@ spec: healthChecks: - apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease - name: cloudflare-ddns + name: cloudflared namespace: networking interval: 30m - timeout: 5m + retryInterval: 1m + timeout: 3m diff --git a/kubernetes/apps/networking/external-dns/app/dnsendpoint-crd.yaml b/kubernetes/apps/networking/external-dns/app/dnsendpoint-crd.yaml new file mode 100644 index 00000000000..9254f89d1eb --- /dev/null +++ b/kubernetes/apps/networking/external-dns/app/dnsendpoint-crd.yaml @@ -0,0 +1,93 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-sigs/external-dns/pull/2007" + creationTimestamp: null + name: dnsendpoints.externaldns.k8s.io +spec: + group: externaldns.k8s.io + names: + kind: DNSEndpoint + listKind: DNSEndpointList + plural: dnsendpoints + singular: dnsendpoint + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DNSEndpointSpec defines the desired state of DNSEndpoint + properties: + endpoints: + items: + description: Endpoint is a high-level way of a connection between a service and an IP + properties: + dnsName: + description: The hostname of the DNS record + type: string + labels: + additionalProperties: + type: string + description: Labels stores labels defined for the Endpoint + type: object + providerSpecific: + description: ProviderSpecific stores provider specific config + items: + description: ProviderSpecificProperty holds the name and value of a configuration which is specific to individual DNS providers + properties: + name: + type: string + value: + type: string + type: object + type: array + recordTTL: + description: TTL for the record + format: int64 + type: integer + recordType: + description: RecordType type of record, e.g. CNAME, A, SRV, TXT etc + type: string + setIdentifier: + description: Identifier to distinguish multiple records with the same name and type (e.g. Route53 records with routing policies other than 'simple') + type: string + targets: + description: The targets the DNS record points to + items: + type: string + type: array + type: object + type: array + type: object + status: + description: DNSEndpointStatus defines the observed state of DNSEndpoint + properties: + observedGeneration: + description: The generation observed by the external-dns controller. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubernetes/apps/networking/external-dns/app/helmrelease.yaml b/kubernetes/apps/networking/external-dns/app/helmrelease.yaml index 61662c599a2..3f48dbf1b35 100644 --- a/kubernetes/apps/networking/external-dns/app/helmrelease.yaml +++ b/kubernetes/apps/networking/external-dns/app/helmrelease.yaml @@ -2,7 +2,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: - name: external-dns + name: &app external-dns namespace: networking spec: interval: 15m @@ -26,8 +26,7 @@ spec: uninstall: keepHistory: false values: - interval: 2m - # logLevel: debug + fullnameOverride: *app provider: cloudflare env: - name: CF_API_EMAIL @@ -41,10 +40,12 @@ spec: name: external-dns-secret key: api-key extraArgs: - - --cloudflare-proxied - --annotation-filter=external-dns.alpha.kubernetes.io/target + - --cloudflare-proxied + - --crd-source-apiversion=externaldns.k8s.io/v1alpha1 + - --crd-source-kind=DNSEndpoint policy: sync - sources: ["ingress"] + sources: ["crd", "ingress"] txtPrefix: k8s. txtOwnerId: default domainFilters: ["${SECRET_DOMAIN}"] diff --git a/kubernetes/apps/networking/external-dns/app/kustomization.yaml b/kubernetes/apps/networking/external-dns/app/kustomization.yaml index a07ca991ddf..1278dd8b558 100644 --- a/kubernetes/apps/networking/external-dns/app/kustomization.yaml +++ b/kubernetes/apps/networking/external-dns/app/kustomization.yaml @@ -3,5 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: networking resources: + - ./dnsendpoint-crd.yaml - ./secret.sops.yaml - ./helmrelease.yaml diff --git a/kubernetes/apps/networking/ingress-nginx/app/cloudflare-networks.txt b/kubernetes/apps/networking/ingress-nginx/app/cloudflare-networks.txt deleted file mode 100644 index d6e3abd1a89..00000000000 --- a/kubernetes/apps/networking/ingress-nginx/app/cloudflare-networks.txt +++ /dev/null @@ -1 +0,0 @@ -173.245.48.0/20\,103.21.244.0/22\,103.22.200.0/22\,103.31.4.0/22\,141.101.64.0/18\,108.162.192.0/18\,190.93.240.0/20\,188.114.96.0/20\,197.234.240.0/22\,198.41.128.0/17\,162.158.0.0/15\,104.16.0.0/13\,104.24.0.0/14\,172.64.0.0/13\,131.0.72.0/22\,2400:cb00::/32\,2606:4700::/32\,2803:f800::/32\,2405:b500::/32\,2405:8100::/32\,2a06:98c0::/29\,2c0f:f248::/32 diff --git a/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml b/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml index 4d41f42eb6b..c9e27450cf1 100644 --- a/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml +++ b/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml @@ -32,7 +32,8 @@ spec: value: "${TIMEZONE}" service: annotations: - metallb.universe.tf/loadBalancerIPs: "${METALLB_INGRESS_ADDR}" + external-dns.alpha.kubernetes.io/hostname: "ipv4.${SECRET_DOMAIN}" + loadBalancerIP: "${METALLB_INGRESS_ADDR}" externalTrafficPolicy: Local publishService: enabled: true @@ -44,28 +45,20 @@ spec: client-body-timeout: 120 custom-http-errors: 400,401,403,404,500,502,503,504 enable-brotli: "true" - forwarded-for-header: "CF-Connecting-IP" hsts-max-age: "31449600" keep-alive: 120 keep-alive-requests: 10000 proxy-body-size: "100M" ssl-protocols: "TLSv1.3 TLSv1.2" - use-forwarded-headers: "true" - log-format-escape-json: "true" - log-format-upstream: >- - {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", - "x_forwarded_for": "$proxy_add_x_forwarded_for", "request_id": "$req_id", - "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time, - "status": $status, "vhost": "$host", "request_proto": "$server_protocol", - "path": "$uri", "request_query": "$args", "request_length": $request_length, - "duration": $request_time,"method": "$request_method", "http_referrer": "$http_referer", - "http_user_agent": "$http_user_agent"} metrics: - enabled: false + enabled: true + serviceMonitor: + enabled: true + namespace: networking + namespaceSelector: + any: true extraArgs: default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls" - podAnnotations: - configmap.reloader.stakater.com/reload: cloudflare-networks resources: requests: cpu: 10m @@ -78,14 +71,5 @@ spec: repository: ghcr.io/tarampampam/error-pages tag: 2.24.0 extraEnvs: - - name: TEMPLATE_NAME - value: lost-in-space - - name: SHOW_DETAILS - value: "false" - valuesFrom: - # Cloudflare Networks - # https://www.cloudflare.com/ips/ - - targetPath: controller.config.proxy-real-ip-cidr - kind: ConfigMap - name: cloudflare-networks - valuesKey: cloudflare-networks.txt + - { name: TEMPLATE_NAME, value: lost-in-space } + - { name: SHOW_DETAILS, value: "false" } diff --git a/kubernetes/apps/networking/ingress-nginx/app/kustomization.yaml b/kubernetes/apps/networking/ingress-nginx/app/kustomization.yaml index bbb7ea265d2..c83d92a87e5 100644 --- a/kubernetes/apps/networking/ingress-nginx/app/kustomization.yaml +++ b/kubernetes/apps/networking/ingress-nginx/app/kustomization.yaml @@ -4,9 +4,3 @@ kind: Kustomization namespace: networking resources: - ./helmrelease.yaml -configMapGenerator: - - name: cloudflare-networks - files: - - ./cloudflare-networks.txt -generatorOptions: - disableNameSuffixHash: true diff --git a/kubernetes/apps/networking/kustomization.yaml b/kubernetes/apps/networking/kustomization.yaml index 4a86c4fdc2b..6b14f9b641e 100644 --- a/kubernetes/apps/networking/kustomization.yaml +++ b/kubernetes/apps/networking/kustomization.yaml @@ -3,7 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - - ./cloudflare-ddns/ks.yaml - ./external-dns/ks.yaml - ./ingress-nginx/ks.yaml - ./k8s-gateway/ks.yaml diff --git a/terraform/cloudflare/main.tf b/terraform/cloudflare/main.tf deleted file mode 100644 index 6467f5e6296..00000000000 --- a/terraform/cloudflare/main.tf +++ /dev/null @@ -1,93 +0,0 @@ -terraform { - - required_providers { - cloudflare = { - source = "cloudflare/cloudflare" - version = "4.4.0" - } - http = { - source = "hashicorp/http" - version = "3.3.0" - } - sops = { - source = "carlpett/sops" - version = "0.7.2" - } - } -} - -data "sops_file" "cloudflare_secrets" { - source_file = "secret.sops.yaml" -} - -provider "cloudflare" { - email = data.sops_file.cloudflare_secrets.data["cloudflare_email"] - api_key = data.sops_file.cloudflare_secrets.data["cloudflare_apikey"] -} - -data "cloudflare_zones" "domain" { - filter { - name = data.sops_file.cloudflare_secrets.data["cloudflare_domain"] - } -} - -resource "cloudflare_zone_settings_override" "cloudflare_settings" { - zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id") - settings { - ssl = "strict" - always_use_https = "on" - min_tls_version = "1.2" - opportunistic_encryption = "on" - tls_1_3 = "zrt" - automatic_https_rewrites = "on" - universal_ssl = "on" - browser_check = "on" - challenge_ttl = 1800 - privacy_pass = "on" - security_level = "medium" - brotli = "on" - minify { - css = "on" - js = "on" - html = "on" - } - rocket_loader = "on" - always_online = "off" - development_mode = "off" - http3 = "on" - zero_rtt = "on" - ipv6 = "on" - websockets = "on" - opportunistic_onion = "on" - pseudo_ipv4 = "off" - ip_geolocation = "on" - email_obfuscation = "on" - server_side_exclude = "on" - hotlink_protection = "off" - security_header { - enabled = false - } - } -} - -data "http" "ipv4" { - url = "http://ipv4.icanhazip.com" -} - -resource "cloudflare_record" "ipv4" { - name = "ipv4" - zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id") - value = chomp(data.http.ipv4.response_body) - proxied = true - type = "A" - ttl = 1 -} - -resource "cloudflare_record" "root" { - name = data.sops_file.cloudflare_secrets.data["cloudflare_domain"] - zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id") - value = "ipv4.${data.sops_file.cloudflare_secrets.data["cloudflare_domain"]}" - proxied = true - type = "CNAME" - ttl = 1 -} diff --git a/tmpl/.sops.yaml b/tmpl/.sops.yaml index 727731e39b9..e00c74f953a 100644 --- a/tmpl/.sops.yaml +++ b/tmpl/.sops.yaml @@ -9,7 +9,3 @@ creation_rules: key_groups: - age: - ${BOOTSTRAP_AGE_PUBLIC_KEY} - - path_regex: terraform/.*\.sops\.ya?ml - key_groups: - - age: - - ${BOOTSTRAP_AGE_PUBLIC_KEY} diff --git a/tmpl/kubernetes/cloudflare-ddns-secret.sops.yaml b/tmpl/kubernetes/cloudflare-ddns-secret.sops.yaml deleted file mode 100644 index d9af99bffbd..00000000000 --- a/tmpl/kubernetes/cloudflare-ddns-secret.sops.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: cloudflare-ddns-secret - namespace: networking -type: Opaque -stringData: - CLOUDFLARE_EMAIL: "${BOOTSTRAP_CLOUDFLARE_EMAIL}" - CLOUDFLARE_APIKEY: "${BOOTSTRAP_CLOUDFLARE_APIKEY}" - CLOUDFLARE_RECORD_NAME: "ipv4.${BOOTSTRAP_CLOUDFLARE_DOMAIN}" diff --git a/tmpl/kubernetes/cloudflared-secret.sops.yaml b/tmpl/kubernetes/cloudflared-secret.sops.yaml new file mode 100644 index 00000000000..7201bde1299 --- /dev/null +++ b/tmpl/kubernetes/cloudflared-secret.sops.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: cloudflared-secret + namespace: networking +data: + TUNNEL_ID: "${BOOTSTRAP_CLOUDFLARE_TUNNEL_ID}" + credentials.json: | + { + "AccountTag": "${BOOTSTRAP_CLOUDFLARE_ACCOUNT_TAG}", + "TunnelSecret": "${BOOTSTRAP_CLOUDFLARE_TUNNEL_SECRET}", + "TunnelID": "${BOOTSTRAP_CLOUDFLARE_TUNNEL_ID}" + } diff --git a/tmpl/kubernetes/cluster-secrets.sops.yaml b/tmpl/kubernetes/cluster-secrets.sops.yaml index 22cc746dd5f..206717798d8 100644 --- a/tmpl/kubernetes/cluster-secrets.sops.yaml +++ b/tmpl/kubernetes/cluster-secrets.sops.yaml @@ -7,3 +7,4 @@ metadata: stringData: SECRET_DOMAIN: "${BOOTSTRAP_CLOUDFLARE_DOMAIN}" SECRET_CLOUDFLARE_EMAIL: "${BOOTSTRAP_CLOUDFLARE_EMAIL}" + SECRET_CLOUDFLARE_TUNNEL_ID: "${BOOTSTRAP_CLOUDFLARE_TUNNEL_ID}" diff --git a/tmpl/terraform/secret.sops.yaml b/tmpl/terraform/secret.sops.yaml deleted file mode 100644 index 117a997ed7c..00000000000 --- a/tmpl/terraform/secret.sops.yaml +++ /dev/null @@ -1,3 +0,0 @@ -cloudflare_email: "${BOOTSTRAP_CLOUDFLARE_EMAIL}" -cloudflare_apikey: "${BOOTSTRAP_CLOUDFLARE_APIKEY}" -cloudflare_domain: "${BOOTSTRAP_CLOUDFLARE_DOMAIN}"