From ff331b64bb89cd5fd5f38d9e584557acaf6cd504 Mon Sep 17 00:00:00 2001 From: marcincuber Date: Sat, 12 Jan 2019 18:45:54 +0000 Subject: [PATCH] Add full example of flux system with other resources --- LICENSE | 220 ++-------------- README.md | 239 +++++++++++++++++- .../namespace-allowed-permissions.yaml | 61 +++++ .../newrelic-token-sealed.yaml | 11 + system/external-dns/external-dns.yaml | 83 ++++++ system/flux-account.yaml | 44 ++++ system/flux-deployment.yaml | 95 +++++++ system/flux-secret.yaml | 7 + system/memcache-dep.yaml | 29 +++ system/memcache-svc.yaml | 15 ++ system/metrics-server/README.md | 117 +++++++++ .../aggregated-metrics-reader.yaml | 17 ++ system/metrics-server/auth-delegator.yaml | 13 + system/metrics-server/auth-reader.yaml | 14 + system/metrics-server/metrics-apiservice.yaml | 14 + .../metrics-server-deployment.yaml | 37 +++ .../metrics-server-service.yaml | 15 ++ system/metrics-server/resource-reader.yaml | 38 +++ system/namespaces/sample-dev.yaml | 10 + ...be-state-metrics-cluster-role-binding.yaml | 13 + .../kube-state-metrics-cluster-role.yaml | 55 ++++ .../kube-state-metrics-deployment.yaml | 59 +++++ .../kube-state-metrics-role-binding.yaml | 14 + .../kubernetes/kube-state-metrics-role.yaml | 22 ++ .../kube-state-metrics-service-account.yaml | 5 + .../kube-state-metrics-service.yaml | 21 ++ .../newrelic-k8s/newrelic-k8s.yaml | 129 ++++++++++ .../sample-dev-flux-project/flux-account.yaml | 42 +++ .../flux-deployment.yaml | 91 +++++++ .../sample-dev-flux-project/flux-secret.yaml | 7 + .../sample-dev-flux-project/memcache-dep.yaml | 29 +++ .../sample-dev-flux-project/memcache-svc.yaml | 15 ++ system/role-bindings/sample-dev-rb.yaml | 32 +++ system/sealed-secrets/controller.yaml | 122 +++++++++ system/sealed-secrets/sealedsecret-crd.yaml | 14 + system/service-accounts/sample-dev-sa.yaml | 6 + 36 files changed, 1553 insertions(+), 202 deletions(-) create mode 100644 system/cluster-roles/namespace-allowed-permissions.yaml create mode 100644 system/encrypted-secrets/newrelic-token-sealed.yaml create mode 100644 system/external-dns/external-dns.yaml create mode 100644 system/flux-account.yaml create mode 100644 system/flux-deployment.yaml create mode 100644 system/flux-secret.yaml create mode 100644 system/memcache-dep.yaml create mode 100644 system/memcache-svc.yaml create mode 100644 system/metrics-server/README.md create mode 100644 system/metrics-server/aggregated-metrics-reader.yaml create mode 100644 system/metrics-server/auth-delegator.yaml create mode 100644 system/metrics-server/auth-reader.yaml create mode 100644 system/metrics-server/metrics-apiservice.yaml create mode 100644 system/metrics-server/metrics-server-deployment.yaml create mode 100644 system/metrics-server/metrics-server-service.yaml create mode 100644 system/metrics-server/resource-reader.yaml create mode 100644 system/namespaces/sample-dev.yaml create mode 100644 system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-cluster-role-binding.yaml create mode 100644 system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-cluster-role.yaml create mode 100644 system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-deployment.yaml create mode 100644 system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-role-binding.yaml create mode 100644 system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-role.yaml create mode 100644 system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-service-account.yaml create mode 100644 system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-service.yaml create mode 100644 system/newrelic-monitoring/newrelic-k8s/newrelic-k8s.yaml create mode 100644 system/projects/test-sub-project/sample-dev-flux-project/flux-account.yaml create mode 100644 system/projects/test-sub-project/sample-dev-flux-project/flux-deployment.yaml create mode 100644 system/projects/test-sub-project/sample-dev-flux-project/flux-secret.yaml create mode 100644 system/projects/test-sub-project/sample-dev-flux-project/memcache-dep.yaml create mode 100644 system/projects/test-sub-project/sample-dev-flux-project/memcache-svc.yaml create mode 100644 system/role-bindings/sample-dev-rb.yaml create mode 100644 system/sealed-secrets/controller.yaml create mode 100644 system/sealed-secrets/sealedsecret-crd.yaml create mode 100644 system/service-accounts/sample-dev-sa.yaml diff --git a/LICENSE b/LICENSE index 261eeb9..8e49eed 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,19 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +The MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index b64cc25..9e1388d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,239 @@ # k8s-flux -Example deployment of resources using flux + +This repository contains code used to define the [Weaveworks Flux](https://github.com/weaveworks/flux) set up for the one of my sample clusters. Flux is used to sync Kubernetes YAML stored in Git with a remote cluster. + +Please note that this repo is only using YAMLs that can be copied across to your projects. I am not a big fan of Helm as it usually takes more time than necessary to deploy resources. Hopefully, you will find it easy and quick to follow my example which I am using with multiple kubernetes clusters. + +## Layout, YAMLs and Flux Configuration + +All of the relevant YAML in this repository are in the `system/` directory. In these directories are the YAMLs to spin up the _system_ Flux in the `kube-system` namespace. Namespaces definitions in `system/namespaces/`. And number of services that can be found useful to deploy through flux. + +### _System_ Flux + +This Flux has a **cluster admin** role so is extremly powerful! The _system_ flux deploys code in the `system/` directory and its child directories for its cluster. Note that this includes the YAML for iteslf! Changes at this level in the directory can break your Flux deployment. The system YAMLs at this level are very similar to the default YAMLs in [Flux](https://github.com/weaveworks/flux). + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flux + namespace: kube-system +spec: + template: + spec: + containers: + - name: flux + args: + - --git-url=git@github.com:marcin/k8s-flux.git + - --git-branch=master + - --git-path=system + - --git-poll-interval=120s + - --registry-poll-interval=120s + - --k8s-namespace-whitelist=kube-system +``` +* `--git-url` is the URL Flux watches for changes. For the _system_ Flux that is this repository +* `--git-branch` is the desired branch of the repository to watch +* `--git-poll` sets the poll interval for Flux to check for changes to git +* `--git-path` is the subdirectory in the target repo that Flux deploys. For the `system` flux that is `system/` +* `--k8s-namespace-whitelist` is the namespace in which Flux watches for deployed containers and poll the remote registry to discover newer versions. + +More flags and further docs can be found -> [Flux](https://github.com/weaveworks/flux). + +### Resources deployed through Flux + +In this example you can find the following Kuberentes resources to be controlled/deployed by flux; + +* Namespaces +* Cluster Roles +* Role Bindings and/or Cluster Role Bindings +* Service Accounts +* External DNS +* Metrics Server +* Sealed-Secrets [home of sealed-secrets](https://github.com/bitnami-labs/sealed-secrets) +* NewRelic Monitoring +* Kube State Metrics required for NewRelic Monitoring +* Flux instance/s for applications + +## External DNS + +External DNS Kubernetes YAML is stored in `system/external-dns/`. Documentation and additional information about [External DNS](https://github.com/kubernetes-incubator/external-dns). + +In my cluster I make use of `kube2iam` which is being used in External DNS deployment template to perform required actions. Please ensure that you replace `ENTER_YOUR_IAM_ROLE_ARN_HERE` in `system/external-dns/external-dns.yaml` with the ARN of the role that exists in your AWS account. + +### Required IAM Permissions for IAM ROLE- policy + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "route53:ChangeResourceRecordSets" + ], + "Resource": [ + "arn:aws:route53:::hostedzone/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "route53:ListHostedZones", + "route53:ListResourceRecordSets" + ], + "Resource": [ + "*" + ] + } + ] +} +``` + +In order to configure the external-dns to your taste you can read -> [external-dns configuration options](https://github.com/kubernetes-incubator/external-dns/blob/master/docs/tutorials/aws.md). + +## Metrics Server + +### User guide + +You can find the user guide in +[the official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug-application-cluster/core-metrics-pipeline/). + +### Design + +The detailed design of the project can be found in the following docs: + +- [Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/resource-metrics-api.md) +- [Metrics Server](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/metrics-server.md) + +For the broader view of monitoring in Kubernetes take a look into +[Monitoring architecture](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/monitoring_architecture.md) + +Note that this is just a sample Metrics Server, nothing has been customised. + +## Sealed Secrets + +We use [sealed-secrets](https://github.com/bitnami-labs/sealed-secrets) which allows us to store Encrypted Secret in the Github repository. The SealedSecret can be decrypted only by the controller running in the target cluster and nobody else (not even the original author) is able to obtain the original Secret from the SealedSecret. + +### Details + +The key certificate (public key portion) is used for sealing secrets, and needs to be available wherever `kubeseal` is going to be used. The certificate is not secret information, although you need to ensure you are using the correct file. You can install kubeseal using `brew install kubeseal` or from source [sealed-secrets](https://github.com/bitnami-labs/sealed-secrets). + +`kubeseal` will fetch the certificate from the controller at runtime (requires secure access to the Kubernetes API server), which is convenient for interactive use. The recommended automation workflow is to store the certificate to local disk with `kubeseal --fetch-cert > mycert.crt`, and use it offline with `kubeseal --cert mycert.crt`. Once sealed-secrets has been deployed to the cluster, you should ensure that you the master key has been backed up. + +```bash +# Usage example: +# Create a json/yaml-encoded Secret somehow: +# (note use of `--dry-run` - this is just a local file!) +kubectl create secret generic mysecret --dry-run --from-literal=foo=bar -o json > mysecret.json +# or +kubectl create secret generic mysecret --dry-run --from-literal=foo=bar -o yaml > mysecret.yaml +# encrypt secret (json) +kubeseal < mysecret.json > mysealedsecret.json +# encrypt secret (yaml) +kubeseal --format yaml < mysecret.json > mysealedsecret.yaml +# or +kubeseal --format yaml < mysecret.yaml > mysealedsecret.yaml +# Output file mysealedsecret.json|yaml is safe to commit +``` + +Please note for the above commands it is sufficient to use the `.crt` which is the public part of the secret. You need to store safely your secret containing both private and public parts which can be retrived using the following command; + +``` +kubectl get secret -n kube-system sealed-secrets-key -o yaml > master.pem +``` + +## NewRelic Monitoring + +To activate the Kubernetes integration, we deploy the newrelic-infra agent onto a Kubernetes cluster as a daemon set and it requires kube-state-metrics. Yaml files can be found in `newrelic-monitoring/` folder. They are deployed using Flux but you need to take care of specifying CLUSTER_NAME and NEWRELIC_KEY, see below what is required. + +To find out how to install NewRelic Monitoring for you cluster manually or for any other documentation see -> [NewRelic K8s Integration docs](https://docs.newrelic.com/docs/integrations/kubernetes-integration/installation/kubernetes-installation-configuration). + +### Configuration + +In case you decide to use NewRelic and the templates provided here, make sure to modify the deployment file `system/newrelic-monitoring/newrelic-k8s/newrelic-k8s.yaml`. The following block should be modified; + +``` +env: + - name: "CLUSTER_NAME" + value: "PUT_YOUR_OWN_CLUSTER_NAME_TO_ADD" + - name: "NRIA_LICENSE_KEY" + valueFrom: + secretKeyRef: + name: newrelic-config + key: newrelic_key +``` + +Note that in my example, I create and pass `NRIA_LICENSE_KEY` as a secret. If you prefer to put your newrelic-key directly use; + +``` +env: + - name: NRIA_LICENSE_KEY + value: YOUR_LICENSE_KEY +``` + +### Sealed Newrelic Token + +To handle `newrelic-key as a secret`, I am utilising the `sealed-secrets` resource. Assuming you have your secret locally, you can encrypt it. + +``` +newrelic-token.yaml # Locally stored secret yaml file containing NewRelic key which is base64 encoded +# Fetch your sealed-secrets .crt (public part) +kubeseal --fetch-cert > cert.crt +# Encrypt your secret using the .crt +kubeseal --cert cert.crt --format yaml < newrelic-token.yaml > newrelic-token-sealed.yaml +``` + +Once you generated your encrypted file ensure that you only commit the encrypted file `newrelic-token-sealed.yaml`. You shouldn't commit `newrelic-token.yaml`. + +The encrypted file `newrelic-token-sealed.yaml` is placed in `encrypted-secrets/`. Make sure you update newrelic-token-sealed.yaml with your own encrypted base64 encoded newrelic token, otherwise your newrelic monitoring won't have a valid token to run with. + +## Flux instance for application/s + +If you are planning to deploy your application using Flux to custom namespace/s. You can create another set of templates for Flux app which will only be allow to performed actions on application's namespace. Such example can be found in `projects/test-sub-project/sample-dev-flux-project/`. + +### _Application_ Fluxes + +The _application_ Fluxes reside in their own namespaces in the cluster. These namespaces are simply `-flux` so for `sample-dev-project` the Flux namespace is `sample-dev-flux`. This namespace just contains Flux, memcached and flux K8s service account. + +Each _application_ Flux has namespace admin over its own namespace and the target namespace. This means it can manage all namespaced resources in those namespaces. This permissions boundary is enforced through K8s RBAC. + +The custom arguments for the _application_ Fluxes are as follows; + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flux + namespace: sample-dev-flux +spec: + template: + spec: + serviceAccount: test-sample-dev-flux + containers: + - name: flux + args: + - --git-url=git@github.com:marcincuber/test-app-flux-repo.git + - --git-branch=master + - --git-poll-interval=1m0s + - --git-path=dev-app + - --k8s-namespace-whitelist=sample-dev +``` + +The only signifcant differences between this config and the _system_ Flux config are that the `--git-url` is now pointing at the application flux repository, `--git-path` corresponds do a relevant directory in that repository and the `--k8s-namespace-whitelist` is now the target namespace for the _application_ Flux. Also in your deployment spec you specify service account `test-sample-dev-flux`. + +Please make sure you modify application so that it points at your application's repository. I only used fake values in my `test-sub-project` deployment flux yaml. + +## Disaster Recovery + +In the event that a cluster is completely lost or needs to be spun up from scratch, the applications can be easily be restored with Flux: + +1. Deploy the _system_ Flux manually using `kubectl apply -f system/`. +1. Get the new _system_ Flux SSH key from the new FLux with `kubectl logs -l name=flux -n kube-system | head` or `fluxctl --k8s-fwd-ns=kube-system identity`. +1. Add the _system_ Flux SSH key (public part) to the _system_ GitHub service account or deploy keys in your repo settings. +1. Your sealed-secrets application (custom resource) will be redeployed as part of flux. However, you will need to make sure that you put the previously generated key. Above named as `master.pem`. +1. Put the secret back before starting the controller - or if the controller was already started, replace the newly-created secret and restart the controller (sealed-secrets). +1. Simply run `kubectl replace -f master.pem` and then `kubectl delete pod -n kube-system -l name=sealed-secrets-controller`. +1. Wait for the application Fluxes to be deployed. +1. Get the new _application_ Flux public SSH key from the pod logs with `kubectl logs -l name=flux -n ${target_ns-flux} | head`. Do this for each application Flux that you have configured. +1. Add the _application_ Flux SSH keys to the _application_ GitHub bot account (remove the old keys if necessary). +1. Wait for the application pods to come up. diff --git a/system/cluster-roles/namespace-allowed-permissions.yaml b/system/cluster-roles/namespace-allowed-permissions.yaml new file mode 100644 index 0000000..4655d73 --- /dev/null +++ b/system/cluster-roles/namespace-allowed-permissions.yaml @@ -0,0 +1,61 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + name: cluster-test-role + name: cluster-test-role +rules: +- apiGroups: + - "" + - extensions + - apps + - batch + - autoscaling + resources: + - deployments + - services + - secrets + - pods + - pods/log + - configmaps + - ingresses + - daemonsets + - cronjobs + - horizontalpodautoscalers + - endpoints + - jobs + - pods/exec + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + - serviceaccounts + verbs: + - get + - create + - update + - delete + - patch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use \ No newline at end of file diff --git a/system/encrypted-secrets/newrelic-token-sealed.yaml b/system/encrypted-secrets/newrelic-token-sealed.yaml new file mode 100644 index 0000000..8e3d848 --- /dev/null +++ b/system/encrypted-secrets/newrelic-token-sealed.yaml @@ -0,0 +1,11 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: newrelic-config + namespace: default +spec: + encryptedData: + newrelic_key: HERE_SHOULD_BE_THE_VALUE_OF_BASE64_ENCODED_NEWRELIC_KEY_ENCRYPTED_USING_CRT_FROM_SEALED_SECRETS + +# Note that this file won't work as you need to generate your own encrypted secret. See the README \ No newline at end of file diff --git a/system/external-dns/external-dns.yaml b/system/external-dns/external-dns.yaml new file mode 100644 index 0000000..dd97ecb --- /dev/null +++ b/system/external-dns/external-dns.yaml @@ -0,0 +1,83 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: external-dns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: external-dns +rules: +- apiGroups: + - "" + resources: + - services + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - nodes + verbs: + - list +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: external-dns-viewer +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-dns +subjects: +- kind: ServiceAccount + name: external-dns + namespace: kube-system +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: external-dns + namespace: kube-system +spec: + strategy: + type: Recreate + template: + metadata: + annotations: + iam.amazonaws.com/role: ENTER_YOUR_IAM_ROLE_ARN_HERE + labels: + app: external-dns + spec: + serviceAccountName: external-dns + containers: + - name: external-dns + image: registry.opensource.zalan.do/teapot/external-dns:latest + args: + - --source=service + - --source=ingress + # - --domain-filter= # will make ExternalDNS see only the hosted zones matching provided domain, omit to process all available hosted zones + - --provider=aws + - --policy=upsert-only # would prevent ExternalDNS from deleting any records, omit to enable full synchronization + - --aws-zone-type=public # only look at public hosted zones (valid values are public, private or no value for both) + - --registry=txt + - --txt-owner-id=my-identifier diff --git a/system/flux-account.yaml b/system/flux-account.yaml new file mode 100644 index 0000000..382cd5a --- /dev/null +++ b/system/flux-account.yaml @@ -0,0 +1,44 @@ +--- +# The service account, cluster roles, and cluster role binding are +# only needed for Kubernetes with role-based access control (RBAC). +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + name: flux + name: flux + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + name: flux + name: flux +rules: + - apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' + - nonResourceURLs: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: + name: flux + name: flux + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flux +subjects: + - kind: ServiceAccount + name: flux + namespace: kube-system diff --git a/system/flux-deployment.yaml b/system/flux-deployment.yaml new file mode 100644 index 0000000..cc74f0b --- /dev/null +++ b/system/flux-deployment.yaml @@ -0,0 +1,95 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flux + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + name: flux + strategy: + type: Recreate + template: + metadata: + annotations: + prometheus.io.port: "3031" # tell prometheus to scrape /metrics endpoint's port. + labels: + name: flux + spec: + serviceAccount: flux + volumes: + - name: git-key + secret: + secretName: flux-git-deploy + defaultMode: 0400 # when mounted read-only, we won't be able to chmod + + # This is a tmpfs used for generating SSH keys. In K8s >= 1.10, + # mounted secrets are read-only, so we need a separate volume we + # can write to. + - name: git-keygen + emptyDir: + medium: Memory + + # The following volume is for using a customised known_hosts + # file, which you will need to do if you host your own git + # repo rather than using github or the like. You'll also need to + # mount it into the container, below. See + # https://github.com/weaveworks/flux/blob/master/site/standalone/setup.md#using-a-private-git-host + # - name: ssh-config + # configMap: + # name: flux-ssh-config + + containers: + - name: flux + # There are no ":latest" images for flux. Find the most recent + # release or image version at https://quay.io/weaveworks/flux + # and replace the tag here. + image: quay.io/weaveworks/flux:1.8.2 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 50m + memory: 64Mi + ports: + - containerPort: 3030 # informational + volumeMounts: + - name: git-key + mountPath: /etc/fluxd/ssh # to match location given in image's /etc/ssh/config + readOnly: true # this will be the case perforce in K8s >=1.10 + - name: git-keygen + mountPath: /var/fluxd/keygen # to match location given in image's /etc/ssh/config + + args: + + # if you deployed memcached in a different namespace to flux, + # or with a different service name, you can supply these + # following two arguments to tell fluxd how to connect to it. + # - --memcached-hostname=memcached.default.svc.cluster.local + # - --memcached-service=memcached + - --memcached-timeout=100ms + - --registry-cache-expiry=30m + + # this must be supplied, and be in the tmpfs (emptyDir) + # mounted above, for K8s >= 1.10 + - --ssh-keygen-dir=/var/fluxd/keygen + + # replace or remove the following URL + - --git-url=git@github.com:marcincuber/k8s-flux.git + - --git-branch=master + - --git-path=system + - --git-poll-interval=120s + - --registry-poll-interval=120s + # include these next two to connect to an "upstream" service + # (e.g., Weave Cloud). The token is particular to the service. + # - --connect=wss://cloud.weave.works/api/flux + # - --token=abc123abc123abc123abc123 + + # serve /metrics endpoint at different port. + # make sure to set prometheus' annotation to scrape the port value. + - --listen-metrics=:3031 + + # list of valid namespaces to export, empty for all namespaces. + # comma separated list of namespaces to monitor for workloads. + - --k8s-namespace-whitelist=kube-system diff --git a/system/flux-secret.yaml b/system/flux-secret.yaml new file mode 100644 index 0000000..61c26b4 --- /dev/null +++ b/system/flux-secret.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: flux-git-deploy + namespace: kube-system +type: Opaque diff --git a/system/memcache-dep.yaml b/system/memcache-dep.yaml new file mode 100644 index 0000000..600083b --- /dev/null +++ b/system/memcache-dep.yaml @@ -0,0 +1,29 @@ +--- +# You can optionally deploy memcache, for the Flux daemon to cache +# container image metadata. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + name: memcached + template: + metadata: + labels: + name: memcached + spec: + containers: + - name: memcached + image: memcached:1.4.25 + imagePullPolicy: IfNotPresent + args: + - -m 64 # Maximum memory to use, in megabytes. 64MB is default. + - -p 11211 # Default port, but being explicit is nice. + # - -vv # Uncomment to get logs of each request and response. + ports: + - name: clients + containerPort: 11211 diff --git a/system/memcache-svc.yaml b/system/memcache-svc.yaml new file mode 100644 index 0000000..7f633fa --- /dev/null +++ b/system/memcache-svc.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: memcached + namespace: kube-system +spec: + # The memcache client uses DNS to get a list of memcached servers and then + # uses a consistent hash of the key to determine which server to pick. + clusterIP: None + ports: + - name: memcached + port: 11211 + selector: + name: memcached diff --git a/system/metrics-server/README.md b/system/metrics-server/README.md new file mode 100644 index 0000000..f8f42f5 --- /dev/null +++ b/system/metrics-server/README.md @@ -0,0 +1,117 @@ +# Kubernetes Metrics Server + +## User guide + +You can find the user guide in +[the official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug-application-cluster/core-metrics-pipeline/). + +## Design + +The detailed design of the project can be found in the following docs: + +- [Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/resource-metrics-api.md) +- [Metrics Server](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/metrics-server.md) + +For the broader view of monitoring in Kubernetes take a look into +[Monitoring architecture](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/monitoring_architecture.md) + +## Manual Deployment + +In order to deploy metrics-server run the following from current directory: + +```console +$ kubectl apply -f . +``` + +## Flags + +Metrics Server supports all the standard Kubernetes API server flags, as +well as the standard Kubernetes `glog` logging flags. The most +commonly-used ones are: + +- `--logtostderr`: log to standard error instead of files in the + container. You generally want this on. + +- `--v=`: set log verbosity. It's generally a good idea to run a log + level 1 or 2 unless you're encountering errors. At log level 10, large + amounts of diagnostic information will be reported, include API request + and response bodies, and raw metric results from Kubelet. + +- `--secure-port=`: set the secure port. If you're not running as + root, you'll want to set this to something other than the default (port + 443). + +- `--tls-cert-file`, `--tls-private-key-file`: the serving certificate and + key files. If not specified, self-signed certificates will be + generated, but it's recommended that you use non-self-signed + certificates in production. + +Additionally, Metrics Server defines a number of flags for configuring its +behavior: + +- `--metric-resolution=`: the interval at which metrics will be + scraped from Kubelets (defaults to 60s). + +- `--kubelet-insecure-tls`: skip verifying Kubelet CA certificates. Not + recommended for production usage, but can be useful in test clusters + with self-signed Kubelet serving certificates. + +- `--kubelet-port`: the port to use to connect to the Kubelet (defaults to + the default secure Kubelet port, 10250). + +- `--kubelet-preferred-address-types`: the order in which to consider + different Kubelet node address types when connecting to Kubelet. + Functions similarly to the flag of the same name on the API server. + +## Autoscaling Algorithm + +The autoscaler is implemented as a control loop. It periodically queries pods +described by `Status.PodSelector` of Scale subresource, and collects their CPU +utilization. Then, it compares the arithmetic mean of the pods' CPU utilization +with the target defined in `Spec.CPUUtilization`, and adjusts the replicas of +the Scale if needed to match the target (preserving condition: MinReplicas <= +Replicas <= MaxReplicas). + +The period of the autoscaler is controlled by the +`--horizontal-pod-autoscaler-sync-period` flag of controller manager. The +default value is 30 seconds. + + +CPU utilization is the recent CPU usage of a pod (average across the last 1 +minute) divided by the CPU requested by the pod. In Kubernetes version 1.1, CPU +usage is taken directly from Heapster. + +The target number of pods is calculated from the following formula: + +``` +TargetNumOfPods = ceil(sum(CurrentPodsCPUUtilization) / Target) +``` + +Starting and stopping pods may introduce noise to the metric (for instance, +starting may temporarily increase CPU). So, after each action, the autoscaler +should wait some time for reliable data. Scale-up can only happen if there was +no rescaling within the last 3 minutes. Scale-down will wait for 5 minutes from +the last rescaling. Moreover any scaling will only be made if: +`avg(CurrentPodsConsumption) / Target` drops below 0.9 or increases above 1.1 +(10% tolerance). Such approach has two benefits: + +* Autoscaler works in a conservative way. If new user load appears, it is +important for us to rapidly increase the number of pods, so that user requests +will not be rejected. Lowering the number of pods is not that urgent. + +* Autoscaler avoids thrashing, i.e.: prevents rapid execution of conflicting +decision if the load is not stable. + +## Relative vs. absolute metrics + +We chose values of the target metric to be relative (e.g. 90% of requested CPU +resource) rather than absolute (e.g. 0.6 core) for the following reason. If we +choose absolute metric, user will need to guarantee that the target is lower +than the request. Otherwise, overloaded pods may not be able to consume more +than the autoscaler's absolute target utilization, thereby preventing the +autoscaler from seeing high enough utilization to trigger it to scale up. This +may be especially troublesome when user changes requested resources for a pod +because they would need to also change the autoscaler utilization threshold. +Therefore, we decided to choose relative metric. For user, it is enough to set +it to a value smaller than 100%, and further changes of requested resources will +not invalidate it. diff --git a/system/metrics-server/aggregated-metrics-reader.yaml b/system/metrics-server/aggregated-metrics-reader.yaml new file mode 100644 index 0000000..9304efa --- /dev/null +++ b/system/metrics-server/aggregated-metrics-reader.yaml @@ -0,0 +1,17 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:aggregated-metrics-reader + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + verbs: + - get + - list + - watch diff --git a/system/metrics-server/auth-delegator.yaml b/system/metrics-server/auth-delegator.yaml new file mode 100644 index 0000000..e3442c5 --- /dev/null +++ b/system/metrics-server/auth-delegator.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/system/metrics-server/auth-reader.yaml b/system/metrics-server/auth-reader.yaml new file mode 100644 index 0000000..f0616e1 --- /dev/null +++ b/system/metrics-server/auth-reader.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/system/metrics-server/metrics-apiservice.yaml b/system/metrics-server/metrics-apiservice.yaml new file mode 100644 index 0000000..08b0530 --- /dev/null +++ b/system/metrics-server/metrics-apiservice.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io +spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 diff --git a/system/metrics-server/metrics-server-deployment.yaml b/system/metrics-server/metrics-server-deployment.yaml new file mode 100644 index 0000000..ad2abaf --- /dev/null +++ b/system/metrics-server/metrics-server-deployment.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-server + namespace: kube-system +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server +spec: + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + name: metrics-server + labels: + k8s-app: metrics-server + spec: + serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} + containers: + - name: metrics-server + image: k8s.gcr.io/metrics-server-amd64:v0.3.1 + imagePullPolicy: Always + volumeMounts: + - name: tmp-dir + mountPath: /tmp + diff --git a/system/metrics-server/metrics-server-service.yaml b/system/metrics-server/metrics-server-service.yaml new file mode 100644 index 0000000..082b00c --- /dev/null +++ b/system/metrics-server/metrics-server-service.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" +spec: + selector: + k8s-app: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: 443 diff --git a/system/metrics-server/resource-reader.yaml b/system/metrics-server/resource-reader.yaml new file mode 100644 index 0000000..b6b1eb3 --- /dev/null +++ b/system/metrics-server/resource-reader.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - deployments + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/system/namespaces/sample-dev.yaml b/system/namespaces/sample-dev.yaml new file mode 100644 index 0000000..2144586 --- /dev/null +++ b/system/namespaces/sample-dev.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sample-dev +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sample-dev-flux diff --git a/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-cluster-role-binding.yaml b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000..4c0bade --- /dev/null +++ b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-cluster-role-binding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics +subjects: +- kind: ServiceAccount + name: kube-state-metrics + namespace: kube-system diff --git a/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-cluster-role.yaml b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-cluster-role.yaml new file mode 100644 index 0000000..49636a0 --- /dev/null +++ b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-cluster-role.yaml @@ -0,0 +1,55 @@ +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: kube-state-metrics +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + - nodes + - pods + - services + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: + - list + - watch +- apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - list + - watch +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - list + - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - list + - watch diff --git a/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-deployment.yaml b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-deployment.yaml new file mode 100644 index 0000000..28a119f --- /dev/null +++ b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1beta2 +# Kubernetes versions after 1.9.0 should use apps/v1 +# Kubernetes versions before 1.8.0 should use apps/v1beta1 or extensions/v1beta1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-state-metrics + replicas: 1 + template: + metadata: + labels: + k8s-app: kube-state-metrics + spec: + serviceAccountName: kube-state-metrics + containers: + - name: kube-state-metrics + image: quay.io/coreos/kube-state-metrics:v1.4.0 + ports: + - name: http-metrics + containerPort: 8080 + - name: telemetry + containerPort: 8081 + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 + - name: addon-resizer + image: k8s.gcr.io/addon-resizer:1.7 + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 30Mi + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: + - /pod_nanny + - --container=kube-state-metrics + - --cpu=100m + - --extra-cpu=1m + - --memory=100Mi + - --extra-memory=2Mi + - --threshold=5 + - --deployment=kube-state-metrics diff --git a/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-role-binding.yaml b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-role-binding.yaml new file mode 100644 index 0000000..517d2c8 --- /dev/null +++ b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-role-binding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: kube-state-metrics + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kube-state-metrics-resizer +subjects: +- kind: ServiceAccount + name: kube-state-metrics + namespace: kube-system diff --git a/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-role.yaml b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-role.yaml new file mode 100644 index 0000000..a080307 --- /dev/null +++ b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-role.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + namespace: kube-system + name: kube-state-metrics-resizer +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - extensions + resources: + - deployments + resourceNames: + - kube-state-metrics + verbs: + - get + - update diff --git a/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-service-account.yaml b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-service-account.yaml new file mode 100644 index 0000000..577153b --- /dev/null +++ b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-state-metrics + namespace: kube-system diff --git a/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-service.yaml b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-service.yaml new file mode 100644 index 0000000..b39f241 --- /dev/null +++ b/system/newrelic-monitoring/kube-state-metrics-release-1.4/kubernetes/kube-state-metrics-service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-state-metrics + namespace: kube-system + labels: + k8s-app: kube-state-metrics + annotations: + prometheus.io/scrape: 'true' +spec: + ports: + - name: http-metrics + port: 8080 + targetPort: http-metrics + protocol: TCP + - name: telemetry + port: 8081 + targetPort: telemetry + protocol: TCP + selector: + k8s-app: kube-state-metrics diff --git a/system/newrelic-monitoring/newrelic-k8s/newrelic-k8s.yaml b/system/newrelic-monitoring/newrelic-k8s/newrelic-k8s.yaml new file mode 100644 index 0000000..6c53681 --- /dev/null +++ b/system/newrelic-monitoring/newrelic-k8s/newrelic-k8s.yaml @@ -0,0 +1,129 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: newrelic + namespace: default +automountServiceAccountToken: true +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: newrelic +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/metrics + - nodes/stats + - nodes/proxy + - pods + - services + verbs: + - get + - list +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - privileged +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: newrelic +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: newrelic +subjects: +- kind: ServiceAccount + name: newrelic + namespace: default +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: newrelic-infra + namespace: default + labels: + app: newrelic-infra +spec: + selector: + matchLabels: + name: newrelic-infra + updateStrategy: + type: RollingUpdate # Only supported in Kubernetes version 1.6 or later. + template: + metadata: + labels: + name: newrelic-infra + annotations: + # Needed for Kubernetes versions prior to 1.6.0, where tolerations were set via annotations. + scheduler.alpha.kubernetes.io/tolerations: | + [{"operator": "Exists", "effect": "NoSchedule"},{"operator": "Exists", "effect": "NoExecute"}] + spec: + serviceAccountName: newrelic + hostNetwork: true # This option is a requirement for the Infrastructure Agent to report the proper hostname in New Relic. + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: newrelic-infra + image: newrelic/infrastructure-k8s:1.3.1 + securityContext: + privileged: true + resources: + limits: + memory: 150M + requests: + cpu: 100m + memory: 30M + volumeMounts: + - mountPath: /host + name: host-volume + readOnly: true + - mountPath: /var/run/docker.sock + name: host-docker-socket + env: + - name: "CLUSTER_NAME" + value: "YOUR_OWN_CLUSTER_NAME_TO_ADD" + - name: "NRIA_LICENSE_KEY" + valueFrom: + secretKeyRef: + name: newrelic-config + key: newrelic_key + - name: "NRIA_VERBOSE" + value: "0" + # - name: "CADVISOR_PORT" # Enable direct connection to cAdvisor by specifying the port. Needed for Kubernetes versions prior to 1.7.6. + # value: "4194" + # - name: "KUBE_STATE_METRICS_URL" # If this value is specified then discovery process for kube-state-metrics endpoint won't be triggered. + # value: "http://172.17.0.3:8080" # This is example value. Only HTTP request is accepted. + - name: "NRIA_DISPLAY_NAME" + valueFrom: + fieldRef: + apiVersion: "v1" + fieldPath: "spec.nodeName" + - name: "NRK8S_NODE_NAME" + valueFrom: + fieldRef: + apiVersion: "v1" + fieldPath: "spec.nodeName" + - name: "NRIA_CUSTOM_ATTRIBUTES" + value: '{"clusterName":"$(CLUSTER_NAME)"}' + - name: "NRIA_PASSTHROUGH_ENVIRONMENT" + value: "KUBERNETES_SERVICE_HOST,KUBERNETES_SERVICE_PORT,CLUSTER_NAME,CADVISOR_PORT,NRK8S_NODE_NAME,KUBE_STATE_METRICS_URL" + volumes: + - name: host-volume + hostPath: + path: / + - name: host-docker-socket + hostPath: + path: /var/run/docker.sock + tolerations: + - operator: "Exists" + effect: "NoSchedule" + - operator: "Exists" + effect: "NoExecute" diff --git a/system/projects/test-sub-project/sample-dev-flux-project/flux-account.yaml b/system/projects/test-sub-project/sample-dev-flux-project/flux-account.yaml new file mode 100644 index 0000000..f003a63 --- /dev/null +++ b/system/projects/test-sub-project/sample-dev-flux-project/flux-account.yaml @@ -0,0 +1,42 @@ +--- +# The service account, cluster roles, and cluster role binding are +# only needed for Kubernetes with role-based access control (RBAC). +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + name: test-sample-dev-flux + name: test-sample-dev-flux + namespace: sample-dev-flux +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + labels: + name: test-sample-dev-flux + name: test-sample-dev-flux + namespace: sample-dev +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flux +subjects: + - kind: ServiceAccount + name: test-sample-dev-flux + namespace: sample-dev-flux +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + labels: + name: test-sample-dev-flux + name: test-sample-dev-flux + namespace: sample-dev-flux +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flux +subjects: + - kind: ServiceAccount + name: test-sample-dev-flux + namespace: sample-dev-flux diff --git a/system/projects/test-sub-project/sample-dev-flux-project/flux-deployment.yaml b/system/projects/test-sub-project/sample-dev-flux-project/flux-deployment.yaml new file mode 100644 index 0000000..b1b7f54 --- /dev/null +++ b/system/projects/test-sub-project/sample-dev-flux-project/flux-deployment.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flux + namespace: sample-dev-flux +spec: + replicas: 1 + selector: + matchLabels: + name: flux + strategy: + type: Recreate + template: + metadata: + annotations: + prometheus.io.port: "3031" # tell prometheus to scrape /metrics endpoint's port. + labels: + name: flux + spec: + serviceAccount: test-sample-dev-flux + volumes: + - name: git-key + secret: + secretName: flux-git-deploy + defaultMode: 0400 # when mounted read-only, we won't be able to chmod + + # This is a tmpfs used for generating SSH keys. In K8s >= 1.10, + # mounted secrets are read-only, so we need a separate volume we + # can write to. + - name: git-keygen + emptyDir: + medium: Memory + + # The following volume is for using a customised known_hosts + # file, which you will need to do if you host your own git + # repo rather than using github or the like. You'll also need to + # mount it into the container, below. See + # https://github.com/weaveworks/flux/blob/master/site/standalone/setup.md#using-a-private-git-host + # - name: ssh-config + # configMap: + # name: flux-ssh-config + + containers: + - name: flux + # There are no ":latest" images for flux. Find the most recent + # release or image version at https://quay.io/weaveworks/flux + # and replace the tag here. + image: quay.io/weaveworks/flux:1.8.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3030 # informational + volumeMounts: + - name: git-key + mountPath: /etc/fluxd/ssh # to match location given in image's /etc/ssh/config + readOnly: true # this will be the case perforce in K8s >=1.10 + - name: git-keygen + mountPath: /var/fluxd/keygen # to match location given in image's /etc/ssh/config + + # Include this if you need to mount a customised known_hosts + # file; you'll also need the volume declared above. + # - name: ssh-config + # mountPath: /root/.ssh + + args: + + # if you deployed memcached in a different namespace to flux, + # or with a different service name, you can supply these + # following two arguments to tell fluxd how to connect to it. + # - --memcached-hostname=memcached.default.svc.cluster.local + # - --memcached-service=memcached + + # this must be supplied, and be in the tmpfs (emptyDir) + # mounted above, for K8s >= 1.10 + - --ssh-keygen-dir=/var/fluxd/keygen + + # replace or remove the following URL + - --git-url=git@github.com:marcincuber/test-app-flux-repo.git + - --git-branch=master + - --git-poll-interval=1m0s + - --git-path=dev-app + # include these next two to connect to an "upstream" service + # (e.g., Weave Cloud). The token is particular to the service. + # - --connect=wss://cloud.weave.works/api/flux + # - --token=abc123abc123abc123abc123 + + # serve /metrics endpoint at different port. + # make sure to set prometheus' annotation to scrape the port value. + - --listen-metrics=:3031 + + - --k8s-namespace-whitelist=sample-dev diff --git a/system/projects/test-sub-project/sample-dev-flux-project/flux-secret.yaml b/system/projects/test-sub-project/sample-dev-flux-project/flux-secret.yaml new file mode 100644 index 0000000..5a1d824 --- /dev/null +++ b/system/projects/test-sub-project/sample-dev-flux-project/flux-secret.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: flux-git-deploy + namespace: sample-dev-flux +type: Opaque diff --git a/system/projects/test-sub-project/sample-dev-flux-project/memcache-dep.yaml b/system/projects/test-sub-project/sample-dev-flux-project/memcache-dep.yaml new file mode 100644 index 0000000..7eb7acb --- /dev/null +++ b/system/projects/test-sub-project/sample-dev-flux-project/memcache-dep.yaml @@ -0,0 +1,29 @@ +--- +# You can optionally deploy memcache, for the Flux daemon to cache +# container image metadata. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached + namespace: sample-dev-flux +spec: + replicas: 1 + selector: + matchLabels: + name: memcached + template: + metadata: + labels: + name: memcached + spec: + containers: + - name: memcached + image: memcached:1.4.25 + imagePullPolicy: IfNotPresent + args: + - -m 64 # Maximum memory to use, in megabytes. 64MB is default. + - -p 11211 # Default port, but being explicit is nice. + # - -vv # Uncomment to get logs of each request and response. + ports: + - name: clients + containerPort: 11211 diff --git a/system/projects/test-sub-project/sample-dev-flux-project/memcache-svc.yaml b/system/projects/test-sub-project/sample-dev-flux-project/memcache-svc.yaml new file mode 100644 index 0000000..1afd2c9 --- /dev/null +++ b/system/projects/test-sub-project/sample-dev-flux-project/memcache-svc.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: memcached + namespace: sample-dev-flux +spec: + # The memcache client uses DNS to get a list of memcached servers and then + # uses a consistent hash of the key to determine which server to pick. + clusterIP: None + ports: + - name: memcached + port: 11211 + selector: + name: memcached diff --git a/system/role-bindings/sample-dev-rb.yaml b/system/role-bindings/sample-dev-rb.yaml new file mode 100644 index 0000000..8e6dd2d --- /dev/null +++ b/system/role-bindings/sample-dev-rb.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + labels: + name: sample-namespaced-bound-role + name: sample-namespaced-bound-role + namespace: sample-dev +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-test-role +subjects: + - kind: User + name: system:serviceaccount:kube-system:sample-dev-admin + namespace: sample-dev +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + labels: + name: sample-namespaced-bound-role + name: sample-namespaced-bound-role + namespace: sample-dev-flux +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-test-role +subjects: + - kind: User + name: system:serviceaccount:kube-system:sample-dev-admin + namespace: sample-dev-flux \ No newline at end of file diff --git a/system/sealed-secrets/controller.yaml b/system/sealed-secrets/controller.yaml new file mode 100644 index 0000000..dc4cf56 --- /dev/null +++ b/system/sealed-secrets/controller.yaml @@ -0,0 +1,122 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sealed-secrets-controller + namespace: kube-system +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: sealed-secrets-controller + namespace: kube-system +spec: + template: + metadata: + labels: + name: sealed-secrets-controller + spec: + containers: + - command: + - controller + image: quay.io/bitnami/sealed-secrets-controller:v0.7.0 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + name: sealed-secrets-controller + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /healthz + port: 8080 + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + serviceAccountName: sealed-secrets-controller +--- +apiVersion: v1 +kind: Service +metadata: + name: sealed-secrets-controller + namespace: kube-system +spec: + ports: + - port: 8080 + selector: + name: sealed-secrets-controller +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: sealed-secrets-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sealed-secrets-key-admin +subjects: +- apiGroup: "" + kind: ServiceAccount + name: sealed-secrets-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: sealed-secrets-key-admin + namespace: kube-system +rules: +- apiGroups: + - "" + resourceNames: + - sealed-secrets-key + resources: + - secrets + verbs: + - get +- apiGroups: + - "" + resources: + - secrets + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: sealed-secrets-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: secrets-unsealer +subjects: +- apiGroup: "" + kind: ServiceAccount + name: sealed-secrets-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: secrets-unsealer +rules: +- apiGroups: + - bitnami.com + resources: + - sealedsecrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - delete diff --git a/system/sealed-secrets/sealedsecret-crd.yaml b/system/sealed-secrets/sealedsecret-crd.yaml new file mode 100644 index 0000000..e4ab6a3 --- /dev/null +++ b/system/sealed-secrets/sealedsecret-crd.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: sealedsecrets.bitnami.com +spec: + group: bitnami.com + names: + kind: SealedSecret + listKind: SealedSecretList + plural: sealedsecrets + singular: sealedsecret + scope: Namespaced + version: v1alpha1 diff --git a/system/service-accounts/sample-dev-sa.yaml b/system/service-accounts/sample-dev-sa.yaml new file mode 100644 index 0000000..cc1ca28 --- /dev/null +++ b/system/service-accounts/sample-dev-sa.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sample-dev-admin + namespace: kube-system