From 1713fc4432a4b74babbf6ec717b558618a1b2d00 Mon Sep 17 00:00:00 2001 From: Lawrence Gripper Date: Fri, 18 Jan 2019 13:47:28 +0000 Subject: [PATCH 1/5] Attempt to run integration test. WIP. --- .travis.yml | 4 +- .../basic_service.tf} | 9 +- _examples/yaml/main.tf | 39 -- kubernetes/provider_test.go | 450 +++++++++--------- kubernetes/resource_kubernetes_yaml_test.go | 44 ++ 5 files changed, 278 insertions(+), 268 deletions(-) rename _examples/{yaml/yamltest.yaml => services/basic_service.tf} (70%) delete mode 100644 _examples/yaml/main.tf create mode 100644 kubernetes/resource_kubernetes_yaml_test.go diff --git a/.travis.yml b/.travis.yml index bfc77bd..f35f473 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ ---- services: - docker language: go +sudo: required go: - "1.10.x" dist: xenial @@ -16,6 +16,8 @@ install: script: - make build-binaries + - bash scipts/startminikube_ci.sh + - make testacc matrix: fast_finish: true diff --git a/_examples/yaml/yamltest.yaml b/_examples/services/basic_service.tf similarity index 70% rename from _examples/yaml/yamltest.yaml rename to _examples/services/basic_service.tf index f5d93d3..5c3894e 100644 --- a/_examples/yaml/yamltest.yaml +++ b/_examples/services/basic_service.tf @@ -1,3 +1,7 @@ +provider "k8sraw" {} + +resource "k8sraw_yaml" "test" { + yaml_body = < Date: Fri, 18 Jan 2019 14:13:13 +0000 Subject: [PATCH 2/5] Fix minikube location --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f35f473..f694e5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ install: script: - make build-binaries - - bash scipts/startminikube_ci.sh + - bash scripts/startminikube_ci.sh - make testacc matrix: From fcfe9dfb9ded2c4c8518e56050edd1cd2dc8d28f Mon Sep 17 00:00:00 2001 From: Lawrence Gripper Date: Sun, 20 Jan 2019 10:15:26 +0000 Subject: [PATCH 3/5] Add test infrastructure to check create and destroy --- .gitignore | 2 +- _examples/services/basic_service.tf | 2 +- kubernetes/provider_test.go | 272 +++--------------- kubernetes/resource_kubernetes_yaml_test.go | 20 +- kubernetes/test-fixtures/binary.data | Bin 3990 -> 0 bytes kubernetes/test-fixtures/binary2.data | Bin 4054 -> 0 bytes kubernetes/test-fixtures/kube-config.yaml | 57 ---- kubernetes/test-infra/README.md | 53 ---- kubernetes/test-infra/gke/README.md | 30 -- kubernetes/test-infra/gke/main.tf | 76 ----- kubernetes/test-infra/kops-on-aws/README.md | 28 -- .../test-infra/kops-on-aws/kops-create.sh | 23 -- .../test-infra/kops-on-aws/kops-delete.sh | 7 - kubernetes/test-infra/kops-on-aws/main.tf | 71 ----- kubernetes/test-infra/kops-on-gce/README.md | 28 -- .../test-infra/kops-on-gce/kops-create.sh | 35 --- .../test-infra/kops-on-gce/kops-delete.sh | 17 -- kubernetes/test-infra/kops-on-gce/main.tf | 69 ----- kubernetes/test-infra/kops-waiter.sh | 35 --- .../test-infra/minikube-on-packet/.gitignore | 2 - .../minikube-on-packet/10-install-kvm.sh | 16 -- .../minikube-on-packet/20-install-minikube.sh | 23 -- .../test-infra/minikube-on-packet/README.md | 23 -- .../test-infra/minikube-on-packet/main.tf | 143 --------- 24 files changed, 43 insertions(+), 989 deletions(-) delete mode 100644 kubernetes/test-fixtures/binary.data delete mode 100644 kubernetes/test-fixtures/binary2.data delete mode 100644 kubernetes/test-fixtures/kube-config.yaml delete mode 100644 kubernetes/test-infra/README.md delete mode 100644 kubernetes/test-infra/gke/README.md delete mode 100644 kubernetes/test-infra/gke/main.tf delete mode 100644 kubernetes/test-infra/kops-on-aws/README.md delete mode 100755 kubernetes/test-infra/kops-on-aws/kops-create.sh delete mode 100755 kubernetes/test-infra/kops-on-aws/kops-delete.sh delete mode 100644 kubernetes/test-infra/kops-on-aws/main.tf delete mode 100644 kubernetes/test-infra/kops-on-gce/README.md delete mode 100755 kubernetes/test-infra/kops-on-gce/kops-create.sh delete mode 100755 kubernetes/test-infra/kops-on-gce/kops-delete.sh delete mode 100644 kubernetes/test-infra/kops-on-gce/main.tf delete mode 100755 kubernetes/test-infra/kops-waiter.sh delete mode 100644 kubernetes/test-infra/minikube-on-packet/.gitignore delete mode 100644 kubernetes/test-infra/minikube-on-packet/10-install-kvm.sh delete mode 100644 kubernetes/test-infra/minikube-on-packet/20-install-minikube.sh delete mode 100644 kubernetes/test-infra/minikube-on-packet/README.md delete mode 100644 kubernetes/test-infra/minikube-on-packet/main.tf diff --git a/.gitignore b/.gitignore index 65b242e..accf208 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,4 @@ website/vendor !command/test-fixtures/**/.terraform/ **/terraform-provider-kubernetes **/terraform-provider-kubernetes-yaml -_examples/yaml/terraform-provider-k8sraw +**/terraform-provider-k8sraw diff --git a/_examples/services/basic_service.tf b/_examples/services/basic_service.tf index 5c3894e..8785bac 100644 --- a/_examples/services/basic_service.tf +++ b/_examples/services/basic_service.tf @@ -5,7 +5,7 @@ resource "k8sraw_yaml" "test" { apiVersion: extensions/v1beta1 kind: Ingress metadata: - name: test-ingress + name: __NAME_HERE__ annotations: nginx.ingress.kubernetes.io/rewrite-target: / azure/frontdoor: enabled diff --git a/kubernetes/provider_test.go b/kubernetes/provider_test.go index ec401d2..51e87be 100644 --- a/kubernetes/provider_test.go +++ b/kubernetes/provider_test.go @@ -1,16 +1,11 @@ package kubernetes import ( - // "errors" - // "os" - // "strings" + "fmt" "testing" - // "github.com/hashicorp/terraform/config" "github.com/hashicorp/terraform/helper/schema" "github.com/hashicorp/terraform/terraform" - // api "k8s.io/api/core/v1" - // metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var testAccProviders map[string]terraform.ResourceProvider @@ -29,241 +24,38 @@ func TestProvider(t *testing.T) { } } -// func TestProvider_impl(t *testing.T) { -// var _ terraform.ResourceProvider = Provider() -// } - -// func TestProvider_configure(t *testing.T) { -// resetEnv := unsetEnv(t) -// defer resetEnv() - -// os.Setenv("KUBECONFIG", "test-fixtures/kube-config.yaml") -// os.Setenv("KUBE_CTX", "gcp") - -// c, err := config.NewRawConfig(map[string]interface{}{}) -// if err != nil { -// t.Fatal(err) -// } -// rc := terraform.NewResourceConfig(c) -// p := Provider() -// err = p.Configure(rc) -// if err != nil { -// t.Fatal(err) -// } -// } - -// func unsetEnv(t *testing.T) func() { -// e := getEnv() - -// if err := os.Unsetenv("KUBECONFIG"); err != nil { -// t.Fatalf("Error unsetting env var KUBECONFIG: %s", err) -// } -// if err := os.Unsetenv("KUBE_CONFIG"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_CONFIG: %s", err) -// } -// if err := os.Unsetenv("KUBE_CTX"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_CTX: %s", err) -// } -// if err := os.Unsetenv("KUBE_CTX_AUTH_INFO"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_CTX_AUTH_INFO: %s", err) -// } -// if err := os.Unsetenv("KUBE_CTX_CLUSTER"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_CTX_CLUSTER: %s", err) -// } -// if err := os.Unsetenv("KUBE_HOST"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_HOST: %s", err) -// } -// if err := os.Unsetenv("KUBE_USER"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_USER: %s", err) -// } -// if err := os.Unsetenv("KUBE_PASSWORD"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_PASSWORD: %s", err) -// } -// if err := os.Unsetenv("KUBE_CLIENT_CERT_DATA"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_CLIENT_CERT_DATA: %s", err) -// } -// if err := os.Unsetenv("KUBE_CLIENT_KEY_DATA"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_CLIENT_KEY_DATA: %s", err) -// } -// if err := os.Unsetenv("KUBE_CLUSTER_CA_CERT_DATA"); err != nil { -// t.Fatalf("Error unsetting env var KUBE_CLUSTER_CA_CERT_DATA: %s", err) -// } - -// return func() { -// if err := os.Setenv("KUBE_CONFIG", e.Config); err != nil { -// t.Fatalf("Error resetting env var KUBE_CONFIG: %s", err) -// } -// if err := os.Setenv("KUBECONFIG", e.Config); err != nil { -// t.Fatalf("Error resetting env var KUBECONFIG: %s", err) -// } -// if err := os.Setenv("KUBE_CTX", e.Config); err != nil { -// t.Fatalf("Error resetting env var KUBE_CTX: %s", err) -// } -// if err := os.Setenv("KUBE_CTX_AUTH_INFO", e.CtxAuthInfo); err != nil { -// t.Fatalf("Error resetting env var KUBE_CTX_AUTH_INFO: %s", err) -// } -// if err := os.Setenv("KUBE_CTX_CLUSTER", e.CtxCluster); err != nil { -// t.Fatalf("Error resetting env var KUBE_CTX_CLUSTER: %s", err) -// } -// if err := os.Setenv("KUBE_HOST", e.Host); err != nil { -// t.Fatalf("Error resetting env var KUBE_HOST: %s", err) -// } -// if err := os.Setenv("KUBE_USER", e.User); err != nil { -// t.Fatalf("Error resetting env var KUBE_USER: %s", err) -// } -// if err := os.Setenv("KUBE_PASSWORD", e.Password); err != nil { -// t.Fatalf("Error resetting env var KUBE_PASSWORD: %s", err) -// } -// if err := os.Setenv("KUBE_CLIENT_CERT_DATA", e.ClientCertData); err != nil { -// t.Fatalf("Error resetting env var KUBE_CLIENT_CERT_DATA: %s", err) -// } -// if err := os.Setenv("KUBE_CLIENT_KEY_DATA", e.ClientKeyData); err != nil { -// t.Fatalf("Error resetting env var KUBE_CLIENT_KEY_DATA: %s", err) -// } -// if err := os.Setenv("KUBE_CLUSTER_CA_CERT_DATA", e.ClusterCACertData); err != nil { -// t.Fatalf("Error resetting env var KUBE_CLUSTER_CA_CERT_DATA: %s", err) -// } -// } -// } - -// func getEnv() *currentEnv { -// e := ¤tEnv{ -// Ctx: os.Getenv("KUBE_CTX_CLUSTER"), -// CtxAuthInfo: os.Getenv("KUBE_CTX_AUTH_INFO"), -// CtxCluster: os.Getenv("KUBE_CTX_CLUSTER"), -// Host: os.Getenv("KUBE_HOST"), -// User: os.Getenv("KUBE_USER"), -// Password: os.Getenv("KUBE_PASSWORD"), -// ClientCertData: os.Getenv("KUBE_CLIENT_CERT_DATA"), -// ClientKeyData: os.Getenv("KUBE_CLIENT_KEY_DATA"), -// ClusterCACertData: os.Getenv("KUBE_CLUSTER_CA_CERT_DATA"), -// } -// if cfg := os.Getenv("KUBE_CONFIG"); cfg != "" { -// e.Config = cfg -// } -// if cfg := os.Getenv("KUBECONFIG"); cfg != "" { -// e.Config = cfg -// } -// return e -// } - -// func testAccPreCheck(t *testing.T) { -// hasFileCfg := (os.Getenv("KUBE_CTX_AUTH_INFO") != "" && os.Getenv("KUBE_CTX_CLUSTER") != "") -// hasStaticCfg := (os.Getenv("KUBE_HOST") != "" && -// os.Getenv("KUBE_USER") != "" && -// os.Getenv("KUBE_PASSWORD") != "" && -// os.Getenv("KUBE_CLIENT_CERT_DATA") != "" && -// os.Getenv("KUBE_CLIENT_KEY_DATA") != "" && -// os.Getenv("KUBE_CLUSTER_CA_CERT_DATA") != "") - -// if !hasFileCfg && !hasStaticCfg { -// t.Fatalf("File config (KUBE_CTX_AUTH_INFO and KUBE_CTX_CLUSTER) or static configuration"+ -// " (%s) must be set for acceptance tests", -// strings.Join([]string{ -// "KUBE_HOST", -// "KUBE_USER", -// "KUBE_PASSWORD", -// "KUBE_CLIENT_CERT_DATA", -// "KUBE_CLIENT_KEY_DATA", -// "KUBE_CLUSTER_CA_CERT_DATA", -// }, ", ")) -// } - -// err := testAccProvider.Configure(terraform.NewResourceConfig(nil)) -// if err != nil { -// t.Fatal(err) -// } -// } - -// func skipIfNoGoogleCloudSettingsFound(t *testing.T) { -// if os.Getenv("GOOGLE_PROJECT") == "" || os.Getenv("GOOGLE_REGION") == "" || os.Getenv("GOOGLE_ZONE") == "" { -// t.Skip("The environment variables GOOGLE_PROJECT, GOOGLE_REGION and GOOGLE_ZONE" + -// " must be set to run Google Cloud tests - skipping") -// } -// } - -// func skipIfNoAwsSettingsFound(t *testing.T) { -// if os.Getenv("AWS_DEFAULT_REGION") == "" || os.Getenv("AWS_ZONE") == "" || os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" { -// t.Skip("The environment variables AWS_DEFAULT_REGION, AWS_ZONE, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY" + -// " must be set to run AWS tests - skipping") -// } -// } - -// func skipIfNoLoadBalancersAvailable(t *testing.T) { -// // TODO: Support AWS ELBs -// isInGke, err := isRunningInGke() -// if err != nil { -// t.Fatal(err) -// } -// if !isInGke { -// t.Skip("The Kubernetes endpoint must come from an environment which supports " + -// "load balancer provisioning for this test to run - skipping") -// } -// } - -// func skipIfNotRunningInGke(t *testing.T) { -// isInGke, err := isRunningInGke() -// if err != nil { -// t.Fatal(err) -// } -// if !isInGke { -// t.Skip("The Kubernetes endpoint must come from GKE for this test to run - skipping") -// } -// } - -// func isRunningInMinikube() (bool, error) { -// node, err := getFirstNode() -// if err != nil { -// return false, err -// } - -// labels := node.GetLabels() -// if v, ok := labels["kubernetes.io/hostname"]; ok && v == "minikube" { -// return true, nil -// } -// return false, nil -// } - -// func isRunningInGke() (bool, error) { -// node, err := getFirstNode() -// if err != nil { -// return false, err -// } - -// labels := node.GetLabels() -// if _, ok := labels["cloud.google.com/gke-nodepool"]; ok { -// return true, nil -// } -// return false, nil -// } +func testAccCheckK8srawDestroy(s *terraform.State) error { + return testAccCheckK8srawStatus(s, 404) +} -// func getFirstNode() (api.Node, error) { -// meta := testAccProvider.Meta() -// if meta == nil { -// return api.Node{}, errors.New("Provider not initialized, unable to get cluster node") -// } -// conn, _ := meta.(KubeProvider)() -// resp, err := conn.CoreV1().Nodes().List(metav1.ListOptions{}) -// if err != nil { -// return api.Node{}, err -// } +func testAccCheckK8srawExists(s *terraform.State) error { + return testAccCheckK8srawStatus(s, 200) +} -// if len(resp.Items) < 1 { -// return api.Node{}, errors.New("Expected at least 1 node, none found") -// } +func testAccCheckK8srawStatus(s *terraform.State, expectedCode int) error { + conn, _ := testAccProvider.Meta().(KubeProvider)() + + for _, rs := range s.RootModule().Resources { + if rs.Type != "k8sraw_yaml" { + continue + } + + result := conn.RESTClient().Get().AbsPath(rs.Primary.ID).Do() + var statusCode int + result.StatusCode(&statusCode) + // Did we get the status code we expected? + if statusCode == expectedCode { + continue + } + + // Another error occured + response, err := result.Get() + if err != nil { + return fmt.Errorf("Failed to get 404 for resource, likely a failure to delete occured: %+v", err) + } + return fmt.Errorf("Response: %+v", response) -// return resp.Items[0], nil -// } + } -// type currentEnv struct { -// Config string -// Ctx string -// CtxAuthInfo string -// CtxCluster string -// Host string -// User string -// Password string -// ClientCertData string -// ClientKeyData string -// ClusterCACertData string -// } + return nil +} diff --git a/kubernetes/resource_kubernetes_yaml_test.go b/kubernetes/resource_kubernetes_yaml_test.go index 67cb932..fc7f448 100644 --- a/kubernetes/resource_kubernetes_yaml_test.go +++ b/kubernetes/resource_kubernetes_yaml_test.go @@ -3,31 +3,29 @@ package kubernetes import ( "fmt" "io/ioutil" + "strings" "testing" "github.com/hashicorp/terraform/helper/acctest" "github.com/hashicorp/terraform/helper/resource" - // "github.com/hashicorp/terraform/helper/schema" - // "github.com/hashicorp/terraform/terraform" - // api "k8s.io/api/networking/v1" - // meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - // kubernetes "k8s.io/client-go/kubernetes" ) -func TestAccKubernetesNetworkPolicy_basic(t *testing.T) { +func TestAccK8srawYamlService_basic(t *testing.T) { // var conf api.NetworkPolicy - name := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(10)) + name := fmt.Sprintf("tf-acc-test-service-%s", acctest.RandString(10)) resource.Test(t, resource.TestCase{ PreCheck: func() {}, - IDRefreshName: "kubernetes_network_policy.test", + IDRefreshName: "k8sraw_yaml.test", Providers: testAccProviders, - // CheckDestroy: testAccCheckKubernetesNetworkPolicyDestroy, + CheckDestroy: testAccCheckK8srawDestroy, Steps: []resource.TestStep{ { Config: testk8sRawYamlNetworking(name), Check: resource.ComposeAggregateTestCheckFunc( - resource.TestCheckResourceAttr("kubernetes_network_policy.test", "metadata.0.annotations.%", "1"), + testAccCheckK8srawExists, + resource.TestCheckResourceAttrSet("k8sraw_yaml.test", "yaml_incluster"), + resource.TestCheckResourceAttrSet("k8sraw_yaml.test", "live_yaml_incluster"), ), }, }, @@ -40,5 +38,5 @@ func testk8sRawYamlNetworking(name string) string { if err != nil { panic(err) } - return string(dat) + return strings.Replace(string(dat), "__NAME_HERE__", name, -1) } diff --git a/kubernetes/test-fixtures/binary.data b/kubernetes/test-fixtures/binary.data deleted file mode 100644 index 60d123375616ace13b9caa5710244add831b95e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3990 zcmcInX>c4@5$@hoE3aiI9O0-Ub`ZIUR%aAfia)aCU5)0iW<$n)lQh5QD$cq zt6ZsqUl=GVRaB9Sgu^jca3DYkBwU7qa3v5D2qc6%K!6ZjTsHaP=Ihy=)$B%q`C(VN zq__KC_t)Lu*WC&$%dX|6@#)05!eRCKHBB@#qu^7?wMZz7ROtP?iYzh=eeDjN><7uxRTApp1u3@pra+v69`Jg&KjGS@rL2j0OtJl!Njqt!zLiO1E3>BzTBaJju7{7daqm7x}^7* z>-6Ua{G9Y_%Rm?*=*4-Tsrf6-V-Joy4upZln{juaxwhd1rmk;+wlHhFx)VApMo*c! z)n&su;JiTI4xHyCaB4I`#j?$ogV#sIDZ(E0%Ez<7^>sYCM(#r06CBHPK4wt{p60HI zPr#g;HGyNfj4$%L9M4!wMnlOdZl&nm_`HK*Z3-QOtCD;Yt`Lh}Zjl-(w1WX7^ie!)Re?OuhsMuUy|j46o%v zA+p?0v`6k!7xzbD#+iE_dd|-TPR$BdS#Ct&v3TT*V0YYWv?_F!ya#337>l&!4R64? z=_w^lhd5Eq!j9#tshI%Ol3P)zSb5s!Vm)e0xCM6m+IL%q@gT)LV2(HKRea@6yZl5K z$rn)RmvvOS3mEVtAUaRJip-sz$lUxw1J1WajkCzTM5hX#+{XGZ6Zn3}=@)tp*L@Ro z88x?(v8#S*m^tN?)2LPoeRaJXL;v%}NXHE#KGpN??eYyopAQ3LUh6NiJb*CQ1oJHv zVY2GD<-lN?9LINitXd?_HgAdK2Kgpzv{i;j1x`;nf!3tZ9wIE?L9Hf2*8*!FMycB> z{$XOQG^L6gKf&9WSvdY0M~o@QT6C55zk{k{z5Fq$muJD0ZSrm4>GGW7O|5oUPPCr2 zzVaX*j>~s78SP&4V29`n`z*B9W+(X`Qqk6ZuRNrE@2{|Mx~;M(e@o%X-!Wy0`s5!O z9R8`o!SOZneW2J7Wh(ZEe4+is$rWs8v<4s9#QbTy zX}9{G$a2FASy1%OV7V9}*WCyk63fp!dlna(t-8J!;z6YfyPS%#S8d(EehISl%g+Gv zMQb1v)*@r_B+<0|RAD~Z(M7vpB-)e`^QtP;s#A8acZKD+jX*;qT999WS7*hU=l}zD zy4`4wHkf$}FzgG`W zN7FxMac8b{CH*sOB&5(BkSx!_$TLK*mp`eIXF8bO;7rDW>rO$#uv?P^mJR-xSZr{5NSl%bM?h3$Z5;p#I1EiF+^Gn6I*w|f+lrk&-kGvV%4 znu+X+ZijvI3(3C)#IKIJJJR-pv+DAr=;I5$)p z9>&g-;r@}Kk+C5&wryajeCg=G*l2lpV9_3{42)hjzBn>GG(0+9vBB}y?z9&Be?fN5 z|BZ|l82+bXV4P8@ZLw}Yao^D+sfpt^Og(hdTaI6T>~Mbb!^KBu$FIHn#>0>A+)e)m Dty<%z diff --git a/kubernetes/test-fixtures/binary2.data b/kubernetes/test-fixtures/binary2.data deleted file mode 100644 index e8968301363bbbe3e4cb75fa1ea682c244900a58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4054 zcmcInX^|v(wtsz4Y`f z*-BBRpNv`-pX3rmV(`ESsEB~5!BX%5MNkyP8x<8XKJ=-&^k}4}(wKem) z|Nr-X@4eqUCM++xmY0$b%SY49nP~_tHxU;&GwGDeEFUAz?et4^*RUKW&d!8Zh_81L zXZBh_jpc*H`B^JiG$Dio#2aT#5LjhDsO~jE&G9`h5^tCe{AH(Pf|>2(!xxO<N#{a!Z<1_{M3tW`&gqU}2A``SrjyQ1^V1Lvh#KJg_+4VsySpG$NnMFrQJi zmgX8Sl7^?eT4;H;X_$t_BFjOd%jH%zeh@L^?(q=%7V4qe{A8O=?rAzcX_ow&DZ&5N zqgl)G!p5V_P7_)hY@!S0Hh?*EZ7{KKYOnza<^bprkuRT6K!=FCIKA5~X=E}Mgn3~=N+QN+SYEI}Z8&3OQ-nV1md|E^>zlapZuuPYp5R!X^D&DuurzydECF+F z#srS#GG63&IG(WNemkE47v&Ox>de~_~gUDn78{2EDs?Nt0nbq zq-mn!xTU~g-WAx=-$gdZL)QYY9|IX$ zb^fEo;As>yPW%*iW9~ux4MYqw$69c;E`E=?k1Og=$%=XbT-qYv0iF)aDc;a*h^0hx zTJ1cK;NqBkPlM6wMvt_$g~A>S&9~8OzK>9}IzzWSs-5v~&~T#7v?za1;mSWShlzUR zpBV`M($a$XR`~%?tcxNQ+e5t2pyJ@?r##!Om(V;9!1rzYS_093nT)5)6QJ})P}+_> zXpUhP#9kgJdWZZ_tNHP$nqxD$nTVpA##VdRl*WaNJZK%Zh2=zA@LBScW+!cei514) zBKPtn(M9rO1@)v-vgLbr+;za9PE0QY@oShkb=BXPN)eOfX|VQv@|0?S8Xll|_=qMZ zRLe|j7<5IN>t4u$q_;cE#R$1>PuQVYe$n35xX@_kwcRKlWU8>kDI0sm)*bCvAWN_O z93Wq_3NoQBQYOz3P07y`<}+=5wgX0@kts1RtDstS4)69Zwj8$}sB1*?@=NgQlsFP? zV8E_dO#}uz<5q*6(5!BG+%p}RW#fg3N9(5+Jsg^-Sl(hhr8dj&+9MuUYgDZAEYVi^ zt$OloTRJf@Y}X1*DIOu15oN3HhNuKy2b*@_EQO8rl;}d#rS6{R?y?N}(}ASzdsW=ZW4ae^w>Wx0OOiCOd$2%-BwgXUktPC6XJ~ zxz%V@@1I56jruBP>fyE(>v%dR4>9)+Tv`LQ2C7v*2%MVl4TRODfv`N_`npc^**=(w z=9AGe!t2{c3(JSjlNXWl;-bHhsE=bBp7bSVMT;nn7ma=OL|fOG9OGTMR2_NBE-Q<~p_ToU4hiI2mBFats3#!?imy*k|?`o8^X+VQLblIB)ya zs^yhpm;32_jr^wG{l>*I;)NZKpT8-JP``tfZl6Zs@{<8AEm2N0lqLeVyA+a!o#la( zVeTy&iR`lOk3Dnq$zKcdZ=Gr{MN@5DUV0m4;y`67jV76p!mIpP%2PJ1pb}y^N0~Av z4_X#$ZgZR)>>nD!l9b`z;lbh2K{L9gZ?JU1NZ;s4X{c|(9xeBcTsXEcJTy2oGFG-h z`R1~<8jAk`bmjjA%8C>JsTe0`v=BF0H@|q_;X|qMqt{G6dfj`EUUcMOe&b`sCuYX3 NyzAP7Pi@~x{{i%;^uquE diff --git a/kubernetes/test-fixtures/kube-config.yaml b/kubernetes/test-fixtures/kube-config.yaml deleted file mode 100644 index 366e6e6..0000000 --- a/kubernetes/test-fixtures/kube-config.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: v1 -kind: Config -preferences: {} -clusters: -- cluster: - certificate-authority-data: ZHVtbXk= - server: https://127.0.0.1 - name: default - -contexts: -- context: - cluster: default - user: azure - name: azure -- context: - cluster: default - user: gcp - name: gcp -- context: - cluster: default - user: oidc - name: oidc - -users: -- name: azure - user: - auth-provider: - config: - access-token: dummy - cmd-args: config config-helper --format=json - cmd-path: /usr/local/Caskroom/google-cloud-sdk/latest/google-cloud-sdk/bin/gcloud - expiry: 2017-06-19T14:02:42Z - expiry-key: '{.credential.token_expiry}' - token-key: '{.credential.access_token}' - name: azure -- name: gcp - user: - auth-provider: - config: - access-token: dummy - cmd-args: config config-helper --format=json - cmd-path: /usr/local/Caskroom/google-cloud-sdk/latest/google-cloud-sdk/bin/gcloud - expiry: 2017-06-19T14:02:42Z - expiry-key: '{.credential.token_expiry}' - token-key: '{.credential.access_token}' - name: gcp -- name: oidc - user: - auth-provider: - config: - access-token: dummy - cmd-args: config config-helper --format=json - cmd-path: /usr/local/Caskroom/google-cloud-sdk/latest/google-cloud-sdk/bin/gcloud - expiry: 2017-06-19T14:02:42Z - expiry-key: '{.credential.token_expiry}' - token-key: '{.credential.access_token}' - name: oidc diff --git a/kubernetes/test-infra/README.md b/kubernetes/test-infra/README.md deleted file mode 100644 index 2404bf2..0000000 --- a/kubernetes/test-infra/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Testing Infrastructure - -Here is where we keep the code of testing infrastructure. We have a few environments here: - - - GKE (Google Container Engine) - - kops @ AWS - - kops @ GCE - - minikube @ Packet - -The goal is to make it **as simple as possible** with **as little maintenance burden** -as possible to spin up a particular version of K8S cluster and run -the whole acceptance test suite against it. - -All tests are intended to run on a TeamCity agent in AWS. - -## FAQ - -### Why not just one environment? - -Some environments may seem redundant. Kubernetes is designed -in a way that users shouldn't notice much difference between environments -and the provider (as well as K8S) should _just work_ in most environments. -However there _are_ differences we have to deal with as maintainers -of this provider. e.g. annotations, labels or volumes. - -### Minikube & SSH tunnel - -Running a VM on bare metal and setting up SSH tunnel to get to it may not be ideal, -but it still presents less maintainenace burden compared to building our own environment -or trying to expose tools which were designed to run locally. - -#### Why not just native minikube? - -Because AWS doesn't support nested virtualization [yet](https://aws.amazon.com/ec2/instance-types/i3/). - -#### Why not just run test agent on Packet? - -Because there is [no Packet plugin for TeamCity](https://plugins.jetbrains.com/search?headline=102-cloud-support&pr_productId=&canRedirectToPlugin=false&showPluginCount=false&tags=Cloud+Support). - -### Why kops? Can't we just build our own Terraform configs? - -Possibly, but that introduces potential maintenance burden -any time a new K8S version is introduced. - -## How - -Spinning up most environments should be as simple as - -``` -terraform apply -var=kubernetes_version=1.6.4 -``` - -See each folder for more specific instructions. diff --git a/kubernetes/test-infra/gke/README.md b/kubernetes/test-infra/gke/README.md deleted file mode 100644 index cbbcded..0000000 --- a/kubernetes/test-infra/gke/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# GKE (Google Container Engine) - -You will need the following environment variables to be set: - - - `GOOGLE_CREDENTIALS` - - `GOOGLE_PROJECT` - - `GOOGLE_REGION` - -See [Google Cloud Provider docs](https://www.terraform.io/docs/providers/google/index.html#configuration-reference) for more details about these variables. - -``` -terraform init -terraform apply -var=kubernetes_version=1.7.12-gke.1 -``` - -## Versions - -See https://cloud.google.com/kubernetes-engine/versioning-and-upgrades#versions_available_for_new_cluster_masters for currently available versions. - -## Exporting K8S variables - -``` -export KUBE_HOST=https://$(terraform output endpoint) -export KUBE_USER=$(terraform output username) -export KUBE_PASSWORD=$(terraform output password) -export KUBE_CLIENT_CERT_DATA="$(terraform output client_certificate_b64 | base64 -d -)" -export KUBE_CLIENT_KEY_DATA="$(terraform output client_key_b64 | base64 -d -)" -export KUBE_CLUSTER_CA_CERT_DATA="$(terraform output cluster_ca_certificate_b64 | base64 -d -)" -export GOOGLE_ZONE=$(terraform output zone) -``` diff --git a/kubernetes/test-infra/gke/main.tf b/kubernetes/test-infra/gke/main.tf deleted file mode 100644 index 18d2c9c..0000000 --- a/kubernetes/test-infra/gke/main.tf +++ /dev/null @@ -1,76 +0,0 @@ -provider "google" { - // Provider settings to be provided via ENV variables -} - -data "google_compute_zones" "available" {} - -resource "random_id" "cluster_name" { - byte_length = 10 -} -resource "random_id" "username" { - byte_length = 14 -} -resource "random_id" "password" { - byte_length = 16 -} - -# See https://cloud.google.com/container-engine/supported-versions -variable "kubernetes_version" {} - -resource "google_container_cluster" "primary" { - name = "tf-acc-test-${random_id.cluster_name.hex}" - zone = "${data.google_compute_zones.available.names[0]}" - initial_node_count = 3 - node_version = "${var.kubernetes_version}" - min_master_version = "${var.kubernetes_version}" - - additional_zones = [ - "${data.google_compute_zones.available.names[1]}" - ] - - master_auth { - username = "${random_id.username.hex}" - password = "${random_id.password.hex}" - } - - node_config { - oauth_scopes = [ - "https://www.googleapis.com/auth/compute", - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring" - ] - } -} - -output "google_zone" { - value = "${data.google_compute_zones.available.names[0]}" -} - -output "endpoint" { - value = "${google_container_cluster.primary.endpoint}" -} - -output "username" { - value = "${google_container_cluster.primary.master_auth.0.username}" -} - -output "password" { - value = "${google_container_cluster.primary.master_auth.0.password}" -} - -output "client_certificate_b64" { - value = "${google_container_cluster.primary.master_auth.0.client_certificate}" -} - -output "client_key_b64" { - value = "${google_container_cluster.primary.master_auth.0.client_key}" -} - -output "cluster_ca_certificate_b64" { - value = "${google_container_cluster.primary.master_auth.0.cluster_ca_certificate}" -} - -output "node_version" { - value = "${google_container_cluster.primary.node_version}" -} diff --git a/kubernetes/test-infra/kops-on-aws/README.md b/kubernetes/test-infra/kops-on-aws/README.md deleted file mode 100644 index c289785..0000000 --- a/kubernetes/test-infra/kops-on-aws/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# kops @ AWS - -You will need the standard AWS environment variables to be set, e.g. - - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - -See [AWS Provider docs](https://www.terraform.io/docs/providers/aws/index.html#configuration-reference) for more details about these variables -and alternatives, like `AWS_PROFILE`. - -`route53_zone` has to be a valid domain (zone in Route 53) which has correctly set and propagated NS records, i.e. it is reachable from outside. - -``` -terraform init -terraform apply -var=kubernetes_version=1.6.13 -var=route53_zone=tfacc.testingdomain.com -``` - -## Exporting K8S variables - -``` -export CLUSTER_NAME=$(terraform output cluster_name) -export KUBE_HOST=$(kubectl config view -o jsonpath="{.clusters[?(@.name == \"${CLUSTER_NAME}\")].cluster.server}") -export KUBE_USER=$(kubectl config view -o jsonpath="{.users[?(@.name == \"${CLUSTER_NAME}\")].user.username}") -export KUBE_PASSWORD=$(kubectl config view -o jsonpath="{.users[?(@.name == \"${CLUSTER_NAME}\")].user.password}") -export KUBE_CLIENT_CERT_DATA="$(kubectl config view --raw=true -o json | jq -r ".users[] | select(.name==\"${CLUSTER_NAME}\") | .user[\"client-certificate-data\"]" | base64 -d -)" -export KUBE_CLIENT_KEY_DATA="$(kubectl config view --raw=true -o json | jq -r ".users[] | select(.name==\"${CLUSTER_NAME}\") | .user[\"client-key-data\"]" | base64 -d -)" -export KUBE_CLUSTER_CA_CERT_DATA="$(kubectl config --raw=true view -o json | jq -r ".clusters[] | select(.name==\"${CLUSTER_NAME}\") | .cluster[\"certificate-authority-data\"]" | base64 -d -)" -``` diff --git a/kubernetes/test-infra/kops-on-aws/kops-create.sh b/kubernetes/test-infra/kops-on-aws/kops-create.sh deleted file mode 100755 index 7038d0d..0000000 --- a/kubernetes/test-infra/kops-on-aws/kops-create.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -export NODE_SIZE=t2.small -export MASTER_SIZE=t2.medium -export KOPS_STATE_STORE="s3://${BUCKET_NAME}" - -aws s3api create-bucket --acl=private --bucket $BUCKET_NAME && \ -kops create cluster --cloud=aws \ - --name=$CLUSTER_NAME \ - --state=$KOPS_STATE_STORE \ - --zones=$ZONES \ - --node-count=2 \ - --kubernetes-version=$KUBERNETES_VERSION \ - --ssh-public-key=${SSH_PUBKEY_PATH} \ - --ssh-access=${IP_ADDRESS}/32 \ - --admin-access=${IP_ADDRESS}/32 \ - --yes - -EXIT_CODE=$? -if [ $EXIT_CODE == 0 ]; then - ../kops-waiter.sh 120 -else - exit $EXIT_CODE -fi diff --git a/kubernetes/test-infra/kops-on-aws/kops-delete.sh b/kubernetes/test-infra/kops-on-aws/kops-delete.sh deleted file mode 100755 index 72afa7a..0000000 --- a/kubernetes/test-infra/kops-on-aws/kops-delete.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -kops delete cluster \ - --name=${CLUSTER_NAME} \ - --state=s3://${BUCKET_NAME} \ - --yes && \ -aws s3 rm s3://${BUCKET_NAME} --recursive && \ -aws s3api delete-bucket --bucket $BUCKET_NAME diff --git a/kubernetes/test-infra/kops-on-aws/main.tf b/kubernetes/test-infra/kops-on-aws/main.tf deleted file mode 100644 index f40554f..0000000 --- a/kubernetes/test-infra/kops-on-aws/main.tf +++ /dev/null @@ -1,71 +0,0 @@ -provider "aws" { } - -variable "route53_zone" { - type = "string" -} - -variable "kubernetes_version" { - type = "string" -} - -variable "s3_bucket_prefix" { - type = "string" - default = "kops-tfacc" -} - -variable "private_ssh_key_filename" { - type = "string" - default = "id_rsa" -} - -resource "random_id" "name" { - byte_length = 8 -} - -locals { - cluster_name = "${random_id.name.hex}.kops.${var.route53_zone}" - bucket_name = "${var.s3_bucket_prefix}-${random_id.name.hex}" - public_ssh_key_location = "${path.module}/${var.private_ssh_key_filename}.pub" -} - -data "aws_availability_zones" "available" {} - -data "http" "ipinfo" { - url = "http://ipinfo.io/ip" -} - -resource "tls_private_key" "ssh" { - algorithm = "RSA" -} - -resource "null_resource" "kops" { - provisioner "local-exec" { - command = < $TMP_CREDS_PATH -export GOOGLE_APPLICATION_CREDENTIALS=$TMP_CREDS_PATH - -# Auth for gsutil/gcloud -gcloud auth activate-service-account $(echo $GOOGLE_CREDENTIALS | jq -r .client_email) --key-file=$TMP_CREDS_PATH -gcloud config set pass_credentials_to_gsutil true - -gsutil mb -l $GOOGLE_REGION -p $GOOGLE_PROJECT gs://${BUCKET_NAME} && \ -kops create cluster --cloud=gce \ - --name=$CLUSTER_NAME \ - --state=$KOPS_STATE_STORE \ - --zones $ZONES \ - --master-zones $ZONES \ - --node-count=2 \ - --project $GOOGLE_PROJECT \ - --image "ubuntu-os-cloud/ubuntu-1604-xenial-v20170202" \ - --kubernetes-version=$KUBERNETES_VERSION \ - --ssh-public-key=${SSH_PUBKEY_PATH} \ - --ssh-access=${IP_ADDRESS}/32 \ - --admin-access=${IP_ADDRESS}/32 \ - --yes - -EXIT_CODE=$? -if [ $EXIT_CODE == 0 ]; then - ../kops-waiter.sh 120 -else - exit $EXIT_CODE -fi diff --git a/kubernetes/test-infra/kops-on-gce/kops-delete.sh b/kubernetes/test-infra/kops-on-gce/kops-delete.sh deleted file mode 100755 index 4bf10e3..0000000 --- a/kubernetes/test-infra/kops-on-gce/kops-delete.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -export KOPS_STATE_STORE="gs://${BUCKET_NAME}" -export KOPS_FEATURE_FLAGS=AlphaAllowGCE - -# Auth for kops -echo $GOOGLE_CREDENTIALS > $TMP_CREDS_PATH -export GOOGLE_APPLICATION_CREDENTIALS=$TMP_CREDS_PATH - -# Auth for gsutil/gcloud -gcloud auth activate-service-account $(echo $GOOGLE_CREDENTIALS | jq -r .client_email) --key-file=$TMP_CREDS_PATH -gcloud config set pass_credentials_to_gsutil true - -kops delete cluster \ - --name=${CLUSTER_NAME} \ - --state=${KOPS_STATE_STORE} \ - --yes && \ -gsutil rm -r ${KOPS_STATE_STORE} diff --git a/kubernetes/test-infra/kops-on-gce/main.tf b/kubernetes/test-infra/kops-on-gce/main.tf deleted file mode 100644 index 2444258..0000000 --- a/kubernetes/test-infra/kops-on-gce/main.tf +++ /dev/null @@ -1,69 +0,0 @@ -variable "kubernetes_version" { - type = "string" -} - -variable "zone_name" { - type = "string" -} - -variable "bucket_prefix" { - type = "string" - default = "kops-tfacc" -} - -variable "private_ssh_key_filename" { - type = "string" - default = "id_rsa" -} - -resource "random_id" "name" { - byte_length = 2 -} - -locals { - cluster_name = "k.${random_id.name.hex}.${var.zone_name}" - bucket_name = "${var.bucket_prefix}-${random_id.name.hex}" - public_ssh_key_location = "${path.module}/${var.private_ssh_key_filename}.pub" - tmp_creds_location = "${path.module}/google-creds.json" -} - -data "google_compute_zones" "available" {} - -data "http" "ipinfo" { - url = "http://ipinfo.io/ip" -} - -resource "null_resource" "kops" { - provisioner "local-exec" { - command = < ${var.private_ssh_key_location} && chmod 600 ${var.private_ssh_key_location}" - } -} - -resource "packet_project" "main" { - name = "minikube-test-${random_id.name.hex}" -} - -resource "packet_ssh_key" "default" { - name = "default" - public_key = "${tls_private_key.ssh.public_key_openssh}" -} - -resource "packet_device" "minikube" { - hostname = "minikube" - plan = "${var.packet_plan}" - facility = "${var.packet_facility}" - operating_system = "centos_7" - billing_cycle = "hourly" - project_id = "${packet_project.main.id}" - - provisioner "file" { - connection { - type = "ssh" - user = "root" - private_key = "${tls_private_key.ssh.private_key_pem}" - } - source = "${path.module}/10-install-kvm.sh" - destination = "/tmp/10-install-kvm.sh" - } - provisioner "file" { - connection { - type = "ssh" - user = "root" - private_key = "${tls_private_key.ssh.private_key_pem}" - } - source = "${path.module}/20-install-minikube.sh" - destination = "/tmp/20-install-minikube.sh" - } - - provisioner "remote-exec" { - connection { - type = "ssh" - user = "root" - private_key = "${tls_private_key.ssh.private_key_pem}" - } - inline = [ - "chmod a+x /tmp/10-install-kvm.sh && chmod a+x /tmp/20-install-minikube.sh", - "/tmp/10-install-kvm.sh | tee /var/log/provisioning-11-kvm.log", - "/tmp/20-install-minikube.sh | tee /var/log/provisioning-20-minikube.log", - "/usr/local/bin/minikube start --kubernetes-version=v${var.kubernetes_version}", - # Extract certs so they can be transfered back to client - "mkdir -p /tmp/${var.dotminikube_path}", - "/usr/local/bin/minikube ip | tr -d \"\n\" > /tmp/client/local-ip.txt", - "cp -r ~/.minikube/{ca.crt,client.crt,client.key} /tmp/${var.dotminikube_path}/", - ] - } - - # Pull certs & local IP so we can connect to minikube - provisioner "local-exec" { - command = "scp -i ${var.private_ssh_key_location} -r -o StrictHostKeyChecking=no root@${self.access_public_ipv4}:/tmp/client ./" - } - - depends_on = ["packet_ssh_key.default"] -} - -# Not a great way to setup an SSH tunnel, but it's the only reasonable one -# until https://github.com/hashicorp/terraform/issues/8367 is a thing -resource "null_resource" "ssh_tunnel" { - provisioner "local-exec" { - command = <./tunnel.stdout.log 2>./tunnel.stderr.log -EOF - } - provisioner "local-exec" { - when = "destroy" - command = "ssh -i ${var.private_ssh_key_location} -S tunnel-ctrl.socket -O exit root@${packet_device.minikube.access_public_ipv4}" - } -} - -output "ip_address" { - value = "${packet_device.minikube.access_public_ipv4}" -} - -output "local_tunnel_port" { - value = "${var.local_tunnel_port}" -} - -output "dotminikube_path" { - value = "${var.dotminikube_path}" -} From 4d65afd305f6b61d6b3a971680fadb31e5a025d4 Mon Sep 17 00:00:00 2001 From: Lawrence Gripper Date: Sun, 20 Jan 2019 10:23:38 +0000 Subject: [PATCH 4/5] Build binaries only after tests pass --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f694e5e..617517f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,9 +15,9 @@ install: - go get github.com/kardianos/govendor script: - - make build-binaries - bash scripts/startminikube_ci.sh - make testacc + - make build-binaries matrix: fast_finish: true From 3995a114d5ee482f93b216595a6a99c67abc83d0 Mon Sep 17 00:00:00 2001 From: Lawrence Gripper Date: Sun, 20 Jan 2019 10:41:08 +0000 Subject: [PATCH 5/5] Add basic ingress into tests --- _examples/ingress/basic_ingress.tf | 21 ++++++ .../{services => service}/basic_service.tf | 0 .../resource_kubernetes_examples_test.go | 66 +++++++++++++++++++ kubernetes/resource_kubernetes_yaml_test.go | 42 ------------ 4 files changed, 87 insertions(+), 42 deletions(-) create mode 100644 _examples/ingress/basic_ingress.tf rename _examples/{services => service}/basic_service.tf (100%) create mode 100644 kubernetes/resource_kubernetes_examples_test.go delete mode 100644 kubernetes/resource_kubernetes_yaml_test.go diff --git a/_examples/ingress/basic_ingress.tf b/_examples/ingress/basic_ingress.tf new file mode 100644 index 0000000..f5d1dc3 --- /dev/null +++ b/_examples/ingress/basic_ingress.tf @@ -0,0 +1,21 @@ +provider "k8sraw" {} + +resource "k8sraw_yaml" "test" { + yaml_body = <