From 24ede6b5b8f1e2cbc85bbe9ef343b30985224f2e Mon Sep 17 00:00:00 2001 From: srinandan Date: Sat, 28 Dec 2024 06:08:10 +0000 Subject: [PATCH 1/9] feat: adds basic and publish flags to create #330 --- internal/client/integrations/integrations.go | 19 +++++- internal/cmd/integrations/apply.go | 2 +- internal/cmd/integrations/create.go | 70 ++++++++++++++++++-- test/sample.json | 8 ++- 4 files changed, 89 insertions(+), 10 deletions(-) diff --git a/internal/client/integrations/integrations.go b/internal/client/integrations/integrations.go index 95392556..7f17f3dc 100644 --- a/internal/client/integrations/integrations.go +++ b/internal/client/integrations/integrations.go @@ -269,7 +269,7 @@ type conditionalFailurePolicy struct { // CreateVersion func CreateVersion(name string, content []byte, overridesContent []byte, snapshot string, - userlabel string, grantPermission bool, + userlabel string, grantPermission bool, basicInfo bool, ) (respBody []byte, err error) { iversion := integrationVersion{} if err = json.Unmarshal(content, &iversion); err != nil { @@ -314,9 +314,24 @@ func CreateVersion(name string, content []byte, overridesContent []byte, snapsho return nil, err } + if basicInfo { + apiclient.ClientPrintHttpResponse.Set(false) + } + u, _ := url.Parse(apiclient.GetBaseIntegrationURL()) u.Path = path.Join(u.Path, "integrations", name, "versions") respBody, err = apiclient.HttpClient(u.String(), string(content)) + + if basicInfo { + var respBasicBody []byte + apiclient.ClientPrintHttpResponse.Set(apiclient.GetCmdPrintHttpResponseSetting()) + if respBasicBody, err = getBasicInfo(respBody); err != nil { + return nil, err + } + apiclient.PrettyPrint(respBasicBody) + return respBasicBody, nil + } + return respBody, err } @@ -1219,7 +1234,7 @@ func uploadAsync(name string, filePath string) error { return err } - if _, err := CreateVersion(name, content, nil, "", "", false); err != nil { + if _, err := CreateVersion(name, content, nil, "", "", false, false); err != nil { return err } diff --git a/internal/cmd/integrations/apply.go b/internal/cmd/integrations/apply.go index f74102df..23aae599 100644 --- a/internal/cmd/integrations/apply.go +++ b/internal/cmd/integrations/apply.go @@ -582,7 +582,7 @@ func processIntegration(overridesFile string, integrationFolder string, clilog.Info.Printf("Create integration %s\n", getFilenameWithoutExtension(integrationNames[0])) respBody, err := integrations.CreateVersion(getFilenameWithoutExtension(integrationNames[0]), - integrationBytes, overridesBytes, "", userLabel, grantPermission) + integrationBytes, overridesBytes, "", userLabel, grantPermission, false) if err != nil { return err } diff --git a/internal/cmd/integrations/create.go b/internal/cmd/integrations/create.go index d26a24b1..d4bd8ff8 100644 --- a/internal/cmd/integrations/create.go +++ b/internal/cmd/integrations/create.go @@ -15,9 +15,12 @@ package integrations import ( + "encoding/json" + "fmt" "internal/apiclient" "internal/client/integrations" "os" + "strings" "github.com/spf13/cobra" ) @@ -30,6 +33,16 @@ var CreateCmd = &cobra.Command{ Args: func(cmd *cobra.Command, args []string) (err error) { cmdProject := cmd.Flag("proj") cmdRegion := cmd.Flag("reg") + configVarsJson := cmd.Flag("config-vars-json").Value.String() + configVarsFile := cmd.Flag("config-vars").Value.String() + + if basic && publish { + return fmt.Errorf("cannot combine basic and publish flags") + } + + if !publish && (configVarsFile != "" || configVarsJson != "") { + return fmt.Errorf("cannot use config-vars and config-vars-json flags when publish is false") + } if err = apiclient.SetRegion(cmdRegion.Value.String()); err != nil { return err @@ -37,9 +50,26 @@ var CreateCmd = &cobra.Command{ return apiclient.SetProjectID(cmdProject.Value.String()) }, RunE: func(cmd *cobra.Command, args []string) (err error) { - var overridesContent []byte + var overridesContent, contents []byte + name := cmd.Flag("name").Value.String() + configVarsJson := cmd.Flag("config-vars-json").Value.String() + configVarsFile := cmd.Flag("config-vars").Value.String() + if configVarsJson == "" { + if configVarsFile != "" { + if _, err := os.Stat(configVarsFile); os.IsNotExist(err) { + return err + } + + contents, err = os.ReadFile(configVarsFile) + if err != nil { + return err + } + } + } else { + contents = []byte(configVarsJson) + } if _, err := os.Stat(integrationFile); os.IsNotExist(err) { return err } @@ -60,18 +90,40 @@ var CreateCmd = &cobra.Command{ } } - _, err = integrations.CreateVersion(name, content, overridesContent, snapshot, userLabel, grantPermission) + if publish { + apiclient.DisableCmdPrintHttpResponse() + } + respBody, err := integrations.CreateVersion(name, content, overridesContent, snapshot, + userLabel, grantPermission, basic) + if err != nil { + return err + } + + if publish { + apiclient.EnableCmdPrintHttpResponse() + var integrationMap map[string]interface{} + err = json.Unmarshal(respBody, &integrationMap) + if err != nil { + return err + } + version := integrationMap["name"].(string)[strings.LastIndex(integrationMap["name"].(string), "/")+1:] + if version != "" { + _, err = integrations.Publish(name, version, contents) + } else { + return fmt.Errorf("unable to extract version id from integration") + } + } return err }, } var ( - integrationFile, overridesFile string - grantPermission bool + integrationFile, overridesFile string + grantPermission, publish, basic bool ) func init() { - var name string + var name, configVars, configVarsJson string CreateCmd.Flags().StringVarP(&name, "name", "n", "", "Integration flow name") @@ -85,6 +137,14 @@ func init() { "", "Integration version userlabel") CreateCmd.Flags().BoolVarP(&grantPermission, "grant-permission", "g", false, "Grant the service account permission for integration triggers; default is false") + CreateCmd.Flags().BoolVarP(&publish, "publish", "", + false, "Publish the integration after successful creation; default is false") + CreateCmd.Flags().BoolVarP(&basic, "basic", "", + false, "Returns version and snapshot only in the response; default is false") + CreateCmd.Flags().StringVarP(&configVars, "config-vars", "", + "", "Path to file containing config variables") + CreateCmd.Flags().StringVarP(&configVarsJson, "config-vars-json", "", + "", "Json string containing the config variables if both Json string and file is present Json string will only be used.") _ = CreateCmd.MarkFlagRequired("name") _ = CreateCmd.MarkFlagRequired("file") diff --git a/test/sample.json b/test/sample.json index ef02ab88..6f2a9db8 100644 --- a/test/sample.json +++ b/test/sample.json @@ -147,6 +147,10 @@ "isTransient": true, "producer": "1_1" } - ] + ], + "cloudLoggingDetails": { + "cloudLoggingSeverity": "INFO", + "enableCloudLogging": true + }, + "databasePersistencePolicy": "DATABASE_PERSISTENCE_DISABLED" } - From a6619abe504fba3da5a885c5286d9762c76449b3 Mon Sep 17 00:00:00 2001 From: srinandan Date: Sat, 28 Dec 2024 06:25:51 +0000 Subject: [PATCH 2/9] chore: improve upload documentation #330 --- internal/cmd/integrations/upload.go | 7 ++++--- test/upload.json | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 test/upload.json diff --git a/internal/cmd/integrations/upload.go b/internal/cmd/integrations/upload.go index 2c221968..f4d0b8ec 100644 --- a/internal/cmd/integrations/upload.go +++ b/internal/cmd/integrations/upload.go @@ -60,10 +60,11 @@ func init() { var name string UploadCmd.Flags().StringVarP(&name, "name", "n", - "", "File containing Integration flow name") + "", "Integration flow name") UploadCmd.Flags().StringVarP(&filePath, "file", "f", - "", "File containing an Integration flow json in stringified format. "+ - "To send json, see integrationcli integrations versions patch") + "", "File containing an Integration flow json in the following format: \n"+ + "{\n\t\"content\": The textproto of the IntegrationVersion,\n\t\"fileFormat\": Must be set to YAML or JSON\n}"+ + "\nFor a sample see ./test/upload.json") _ = UploadCmd.MarkFlagRequired("file") } diff --git a/test/upload.json b/test/upload.json new file mode 100644 index 00000000..9f3a0959 --- /dev/null +++ b/test/upload.json @@ -0,0 +1,4 @@ +{ + "content": "{\"snapshotNumber\": \"4\",\"triggerConfigs\": [{\"label\": \"API Trigger\",\"startTasks\": [{\"taskId\": \"1\"}],\"properties\": {\"Trigger name\": \"sample_API_1\"},\"triggerType\": \"API\",\"triggerNumber\": \"1\",\"triggerId\": \"api_trigger/sample_API_1\"}],\"taskConfigs\": [{\"task\": \"GenericRestV2Task\",\"taskId\": \"1\",\"parameters\": {\"throwError\": {\"key\": \"throwError\",\"value\": {\"booleanValue\": true}},\"responseBody\": {\"key\": \"responseBody\",\"value\": {\"stringArray\": {\"stringValues\": [\"$`Task_1_responseBody`$\"]}}},\"disableSSLValidation\": {\"key\": \"disableSSLValidation\",\"value\": {\"booleanValue\": false}},\"httpParams\": {\"key\": \"httpParams\"},\"responseHeader\": {\"key\": \"responseHeader\",\"value\": {\"stringArray\": {\"stringValues\": [\"$`Task_1_responseHeader`$\"]}}},\"userAgent\": {\"key\": \"userAgent\",\"value\": {\"stringValue\": \"\"}},\"httpMethod\": {\"key\": \"httpMethod\",\"value\": {\"stringValue\": \"GET\"}},\"responseStatus\": {\"key\": \"responseStatus\",\"value\": {\"stringArray\": {\"stringValues\": [\"$`Task_1_responseStatus`$\"]}}},\"url\": {\"key\": \"url\",\"value\": {\"stringValue\": \"https://httpbin.org/get\"}},\"urlFetchingService\": {\"key\": \"urlFetchingService\",\"value\": {\"stringValue\": \"HARPOON\"}},\"useSSL\": {\"key\": \"useSSL\",\"value\": {\"booleanValue\": false}},\"requestorId\": {\"key\": \"requestorId\",\"value\": {\"stringValue\": \"\"}},\"urlQueryStrings\": {\"key\": \"urlQueryStrings\"},\"requestBody\": {\"key\": \"requestBody\",\"value\": {\"stringValue\": \"\"}},\"followRedirects\": {\"key\": \"followRedirects\",\"value\": {\"booleanValue\": true}},\"additionalHeaders\": {\"key\": \"additionalHeaders\"}},\"taskExecutionStrategy\": \"WHEN_ALL_SUCCEED\",\"displayName\": \"Call httpbin service\"}],\"integrationParameters\": [{\"key\": \"`Task_1_responseHeader`\",\"dataType\": \"STRING_VALUE\",\"displayName\": \"`Task_1_responseHeader`\",\"isTransient\": true,\"producer\": \"1_1\"},{\"key\": \"`Task_1_responseBody`\",\"dataType\": \"STRING_VALUE\",\"displayName\": \"`Task_1_responseBody`\",\"isTransient\": true,\"producer\": \"1_1\"},{\"key\": \"`Task_1_responseStatus`\",\"dataType\": \"STRING_VALUE\",\"displayName\": \"`Task_1_responseStatus`\",\"isTransient\": true,\"producer\": \"1_1\"}],\"cloudLoggingDetails\": {\"cloudLoggingSeverity\": \"INFO\",\"enableCloudLogging\": true},\"databasePersistencePolicy\": \"DATABASE_PERSISTENCE_DISABLED\"}", + "fileFormat": "JSON" +} From 21e883353b42edf8070965ce2f99c507d20e3073 Mon Sep 17 00:00:00 2001 From: srinandan Date: Sat, 28 Dec 2024 18:38:28 +0000 Subject: [PATCH 3/9] chore: add samples to integrationcli #330 --- README.md | 4 + docs/docs.go | 74 +++++++- internal/cmd/integrations/apply.go | 4 + internal/cmd/integrations/create.go | 3 + internal/cmd/integrations/integrations.go | 20 ++ internal/cmd/integrations/listversions.go | 2 + internal/cmd/integrations/scaffold.go | 4 + samples/sample.json | 156 ++++++++++++++++ samples/sample_overrides.json | 22 +++ samples/scaffold-sample/clouddeploy.yaml | 28 +++ samples/scaffold-sample/deploy.sh | 2 + .../dev/config-variables/sample-config.json | 3 + .../dev/overrides/overrides.json | 10 + samples/scaffold-sample/release.sh | 2 + samples/scaffold-sample/skaffold.yaml | 25 +++ samples/scaffold-sample/src/sample1.json | 175 ++++++++++++++++++ samples/upload.json | 4 + 17 files changed, 536 insertions(+), 2 deletions(-) create mode 100644 samples/sample.json create mode 100644 samples/sample_overrides.json create mode 100644 samples/scaffold-sample/clouddeploy.yaml create mode 100644 samples/scaffold-sample/deploy.sh create mode 100644 samples/scaffold-sample/dev/config-variables/sample-config.json create mode 100644 samples/scaffold-sample/dev/overrides/overrides.json create mode 100644 samples/scaffold-sample/release.sh create mode 100644 samples/scaffold-sample/skaffold.yaml create mode 100644 samples/scaffold-sample/src/sample1.json create mode 100644 samples/upload.json diff --git a/README.md b/README.md index b6f46185..8deed144 100644 --- a/README.md +++ b/README.md @@ -274,6 +274,10 @@ base64 ./test/enc_password.txt > ./test/b64_enc_password.txt # on MacOS, use bas Please refer to this [article](https://www.googlecloudcommunity.com/gc/Cloud-Product-Articles/CI-CD-for-Application-Integration/ta-p/722811) in Google Cloud Community for how to perform CICD in Application Integration with `integrationcli` +## Samples + +Please see [here](./samples/README.md) + ## How do I verify the binary? All artifacts are signed by [cosign](https://github.com/sigstore/cosign). We recommend verifying any artifact before using them. diff --git a/docs/docs.go b/docs/docs.go index 5f926992..5c531d3b 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -15,16 +15,86 @@ package main import ( + "io" "log" + "os" + "path/filepath" + "strings" cmd "internal/cmd" + integrations "internal/cmd/integrations" "github.com/spf13/cobra/doc" + + apiclient "internal/apiclient" ) +const ENABLED = "true" + +var samples = `# integrationcli command Samples + +The following table contains some examples of integrationcli. + +Set up integrationcli with preferences: ` + getSingleLine("integrationcli prefs set -p $project -r $region") + ` + +| Operations | Command | +|---|---| +| integrations | ` + getSingleLine(integrations.GetExample(0)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(1)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(2)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(3)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(4)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(5)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(6)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(7)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(8)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(9)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(10)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(11)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(12)) + `| + + +NOTE: This file is auto-generated during a release. Do not modify.` + func main() { - err := doc.GenMarkdownTree(cmd.RootCmd, "./docs") + var err error + var docFiles []string + + if os.Getenv("INTEGRATIONCLI_SKIP_DOCS") != ENABLED { + + if docFiles, err = filepath.Glob("./docs/integrationcli*.md"); err != nil { + log.Fatal(err) + } + + for _, docFile := range docFiles { + if err = os.Remove(docFile); err != nil { + log.Fatal(err) + } + } + + if err = doc.GenMarkdownTree(cmd.RootCmd, "./docs"); err != nil { + log.Fatal(err) + } + } + + _ = apiclient.WriteByteArrayToFile("./samples/README.md", false, []byte(samples)) +} + +func WriteFile() (byteValue []byte, err error) { + userFile, err := os.Open("./samples/README.md") if err != nil { - log.Fatal(err) + return nil, err } + + defer userFile.Close() + + byteValue, err = io.ReadAll(userFile) + if err != nil { + return nil, err + } + return byteValue, err +} + +func getSingleLine(s string) string { + return "`" + strings.ReplaceAll(strings.ReplaceAll(s, "\\", ""), "\n", "") + "`" } diff --git a/internal/cmd/integrations/apply.go b/internal/cmd/integrations/apply.go index 23aae599..133a5e4a 100644 --- a/internal/cmd/integrations/apply.go +++ b/internal/cmd/integrations/apply.go @@ -143,6 +143,10 @@ var ApplyCmd = &cobra.Command{ return err }, + Example: `Apply scaffold configuration and wait for connectors: ` + GetExample(9) + ` +Apply scaffold configuration for a specific environment: ` + GetExample(10) + ` +Apply scaffold configuration and grant permissions to the service account: ` + GetExample(11) + ` +Apply scaffold configuration, but skip connectors: ` + GetExample(12), } var serviceAccountName, serviceAccountProject, encryptionKey, pipeline, release, outputGCSPath string diff --git a/internal/cmd/integrations/create.go b/internal/cmd/integrations/create.go index d4bd8ff8..4326ac19 100644 --- a/internal/cmd/integrations/create.go +++ b/internal/cmd/integrations/create.go @@ -115,6 +115,9 @@ var CreateCmd = &cobra.Command{ } return err }, + Example: `Create a new Inegration Version with a user label: ` + GetExample(0) + ` +Create a new Inegration Version with overrides: ` + GetExample(1) + ` +Create a new Inegration Version and publish it: ` + GetExample(2), } var ( diff --git a/internal/cmd/integrations/integrations.go b/internal/cmd/integrations/integrations.go index a548c641..e981fba5 100644 --- a/internal/cmd/integrations/integrations.go +++ b/internal/cmd/integrations/integrations.go @@ -27,6 +27,22 @@ var Cmd = &cobra.Command{ var userLabel, snapshot string +var examples = []string{ + `integrationcli integrations create -n $name -f samples/sample.json -u $userLabel --default-token`, + `integrationcli integrations create -n $name -f samples/sample.json -o samples/sample_overrides.json --default-token`, + `integrationcli integrations create -n $name -f samples/sample.json --publish=true --default-token`, + `integrationcli integrations versions list -n $integration --basic=true --default-token`, + `integrationcli integrations versions list -n $integration --basic=true --filter=state=ACTIVE --default-token`, + `integrationcli integrations scaffold -n $integration -s $snapshot -f . --env=dev --default-token`, + `integrationcli integrations scaffold -n $integration -s $snapshot -f . --skip-connectors=true --default-token`, + `integrationcli integrations scaffold -n $integration -s $snapshot -f . --cloud-build=true --default-token`, + `integrationcli integrations scaffold -n $integration -s $snapshot -f . --cloud-deploy=true --default-token`, + `integrationcli integrations apply -f . --wait=true --default-token`, + `integrationcli integrations apply -f . --env=dev --default-token`, + `integrationcli integrations apply -f . --grant-permission=true --default-token`, + `integrationcli integrations apply -f . --skip-connectors=true --default-token`, +} + func init() { var project, region string @@ -49,3 +65,7 @@ func init() { Cmd.AddCommand(ScaffoldCmd) Cmd.AddCommand(ApplyCmd) } + +func GetExample(i int) string { + return examples[i] +} diff --git a/internal/cmd/integrations/listversions.go b/internal/cmd/integrations/listversions.go index b72fec2c..3f74fcfc 100644 --- a/internal/cmd/integrations/listversions.go +++ b/internal/cmd/integrations/listversions.go @@ -46,6 +46,8 @@ var ListVerCmd = &cobra.Command{ false, false, basic) return err }, + Example: `Return a list of versions with basic information: ` + GetExample(3) + ` +Return the version that is published: ` + GetExample(4), } func init() { diff --git a/internal/cmd/integrations/scaffold.go b/internal/cmd/integrations/scaffold.go index f357ce1b..a6e567d1 100644 --- a/internal/cmd/integrations/scaffold.go +++ b/internal/cmd/integrations/scaffold.go @@ -366,6 +366,10 @@ var ScaffoldCmd = &cobra.Command{ return err }, + Example: `Generate scaffold for dev env using snapshot: ` + GetExample(5) + ` +Generate scaffold for integration, but skip connectors: ` + GetExample(6) + ` +Generate scaffold for integration and produce cloud build config: ` + GetExample(7) + ` +Generate scaffold for integration and produce cloud deploy config: ` + GetExample(8), } var ( diff --git a/samples/sample.json b/samples/sample.json new file mode 100644 index 00000000..6f2a9db8 --- /dev/null +++ b/samples/sample.json @@ -0,0 +1,156 @@ +{ + "snapshotNumber": "4", + "triggerConfigs": [ + { + "label": "API Trigger", + "startTasks": [ + { + "taskId": "1" + } + ], + "properties": { + "Trigger name": "sample_API_1" + }, + "triggerType": "API", + "triggerNumber": "1", + "triggerId": "api_trigger/sample_API_1" + } + ], + "taskConfigs": [ + { + "task": "GenericRestV2Task", + "taskId": "1", + "parameters": { + "throwError": { + "key": "throwError", + "value": { + "booleanValue": true + } + }, + "responseBody": { + "key": "responseBody", + "value": { + "stringArray": { + "stringValues": [ + "$`Task_1_responseBody`$" + ] + } + } + }, + "disableSSLValidation": { + "key": "disableSSLValidation", + "value": { + "booleanValue": false + } + }, + "httpParams": { + "key": "httpParams" + }, + "responseHeader": { + "key": "responseHeader", + "value": { + "stringArray": { + "stringValues": [ + "$`Task_1_responseHeader`$" + ] + } + } + }, + "userAgent": { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + "httpMethod": { + "key": "httpMethod", + "value": { + "stringValue": "GET" + } + }, + "responseStatus": { + "key": "responseStatus", + "value": { + "stringArray": { + "stringValues": [ + "$`Task_1_responseStatus`$" + ] + } + } + }, + "url": { + "key": "url", + "value": { + "stringValue": "https://httpbin.org/get" + } + }, + "urlFetchingService": { + "key": "urlFetchingService", + "value": { + "stringValue": "HARPOON" + } + }, + "useSSL": { + "key": "useSSL", + "value": { + "booleanValue": false + } + }, + "requestorId": { + "key": "requestorId", + "value": { + "stringValue": "" + } + }, + "urlQueryStrings": { + "key": "urlQueryStrings" + }, + "requestBody": { + "key": "requestBody", + "value": { + "stringValue": "" + } + }, + "followRedirects": { + "key": "followRedirects", + "value": { + "booleanValue": true + } + }, + "additionalHeaders": { + "key": "additionalHeaders" + } + }, + "taskExecutionStrategy": "WHEN_ALL_SUCCEED", + "displayName": "Call httpbin service" + } + ], + "integrationParameters": [ + { + "key": "`Task_1_responseHeader`", + "dataType": "STRING_VALUE", + "displayName": "`Task_1_responseHeader`", + "isTransient": true, + "producer": "1_1" + }, + { + "key": "`Task_1_responseBody`", + "dataType": "STRING_VALUE", + "displayName": "`Task_1_responseBody`", + "isTransient": true, + "producer": "1_1" + }, + { + "key": "`Task_1_responseStatus`", + "dataType": "STRING_VALUE", + "displayName": "`Task_1_responseStatus`", + "isTransient": true, + "producer": "1_1" + } + ], + "cloudLoggingDetails": { + "cloudLoggingSeverity": "INFO", + "enableCloudLogging": true + }, + "databasePersistencePolicy": "DATABASE_PERSISTENCE_DISABLED" +} diff --git a/samples/sample_overrides.json b/samples/sample_overrides.json new file mode 100644 index 00000000..cc9c2ff2 --- /dev/null +++ b/samples/sample_overrides.json @@ -0,0 +1,22 @@ +{ + "trigger_overrides":[{ + "triggerType": "API", + "triggerNumber": "1", + "apiPath": "nandan_API_1", + "properties": { + "Trigger name": "nandan_API_1" + } + }], + "task_overrides": [{ + "taskId": "1", + "task": "GenericRestV2Task", + "parameters": { + "url": { + "key": "url", + "value": { + "stringValue": "https://httpbin.org/ip" + } + } + } + }] +} \ No newline at end of file diff --git a/samples/scaffold-sample/clouddeploy.yaml b/samples/scaffold-sample/clouddeploy.yaml new file mode 100644 index 00000000..920adfa1 --- /dev/null +++ b/samples/scaffold-sample/clouddeploy.yaml @@ -0,0 +1,28 @@ +apiVersion: deploy.cloud.google.com/v1 +kind: DeliveryPipeline +metadata: + name: app-integration-pipeline +serialPipeline: + stages: + - targetId: sample-env +--- + +apiVersion: deploy.cloud.google.com/v1 +kind: Target +metadata: + name: sample-env +customTarget: + customTargetType: app-integration-target +--- + +apiVersion: deploy.cloud.google.com/v1 +kind: CustomTargetType +metadata: + name: app-integration-target +customActions: + renderAction: render-app-integration + deployAction: deploy-app-integration +deployParameters: + customTarget/Project: $PROJECT_ID + customTarget/Location: $REGION + customTarget/Env: dev diff --git a/samples/scaffold-sample/deploy.sh b/samples/scaffold-sample/deploy.sh new file mode 100644 index 00000000..6a3e79ef --- /dev/null +++ b/samples/scaffold-sample/deploy.sh @@ -0,0 +1,2 @@ +#!/bin/sh +gcloud deploy apply --file=clouddeploy.yaml --region=${LOCATION} --project=${PROJECT_ID} diff --git a/samples/scaffold-sample/dev/config-variables/sample-config.json b/samples/scaffold-sample/dev/config-variables/sample-config.json new file mode 100644 index 00000000..f6891242 --- /dev/null +++ b/samples/scaffold-sample/dev/config-variables/sample-config.json @@ -0,0 +1,3 @@ +{ + "`CONFIG_url`": "https://httpbin.org/get" +} diff --git a/samples/scaffold-sample/dev/overrides/overrides.json b/samples/scaffold-sample/dev/overrides/overrides.json new file mode 100644 index 00000000..f552014a --- /dev/null +++ b/samples/scaffold-sample/dev/overrides/overrides.json @@ -0,0 +1,10 @@ +{ + "integration_overrides": { + "databasePersistencePolicy": "DATABASE_PERSISTENCE_POLICY_UNSPECIFIED", + "enableVariableMasking": false, + "cloudLoggingDetails": { + "cloudLoggingSeverity": "CLOUD_LOGGING_SEVERITY_UNSPECIFIED", + "enableCloudLogging": false + } + } +} diff --git a/samples/scaffold-sample/release.sh b/samples/scaffold-sample/release.sh new file mode 100644 index 00000000..cf602dbb --- /dev/null +++ b/samples/scaffold-sample/release.sh @@ -0,0 +1,2 @@ +#!/bin/sh +gcloud deploy releases create test-release-$((RANDOM % 900 + 100)) --project=${PROJECT_ID} --region=${LOCATION} --delivery-pipeline=app-integration-pipeline diff --git a/samples/scaffold-sample/skaffold.yaml b/samples/scaffold-sample/skaffold.yaml new file mode 100644 index 00000000..36ec7ffb --- /dev/null +++ b/samples/scaffold-sample/skaffold.yaml @@ -0,0 +1,25 @@ +apiVersion: skaffold/v4beta7 +kind: Config +customActions: +- name: render-app-integration + containers: + - name: render + image: gcr.io/google.com/cloudsdktool/google-cloud-cli@sha256:66e2681aa3099b4e517e4cdcdefff8f2aa45d305007124ccdc09686f6712d018 + command: ['/bin/bash'] + args: + - '-c' + - |- + echo "Sample manifest rendered content" > manifest.txt + gsutil cp manifest.txt $CLOUD_DEPLOY_OUTPUT_GCS_PATH/manifest.txt + echo {\"resultStatus\": \"SUCCEEDED\", \"manifestFile\": \"$CLOUD_DEPLOY_OUTPUT_GCS_PATH/manifest.txt\"} > results.json + gsutil cp results.json $CLOUD_DEPLOY_OUTPUT_GCS_PATH/results.json +- name: deploy-app-integration + containers: + - name: deploy + #image: us-docker.pkg.dev/appintegration-toolkit/images/integrationcli:latest + image: us-docker.pkg.dev/nandanks-151422/images/integrationcli-deploy:debug + command: ['sh'] + args: + - '-c' + - |- + integrationcli integrations apply --env=dev --reg=$CLOUD_DEPLOY_LOCATION --proj=$CLOUD_DEPLOY_PROJECT --pipeline=$CLOUD_DEPLOY_DELIVERY_PIPELINE --release=$CLOUD_DEPLOY_RELEASE --output-gcs-path=$CLOUD_DEPLOY_OUTPUT_GCS_PATH --metadata-token diff --git a/samples/scaffold-sample/src/sample1.json b/samples/scaffold-sample/src/sample1.json new file mode 100644 index 00000000..9128eca0 --- /dev/null +++ b/samples/scaffold-sample/src/sample1.json @@ -0,0 +1,175 @@ +{ + "snapshotNumber": "1", + "triggerConfigs": [ + { + "label": "API Trigger", + "triggerType": "API", + "triggerNumber": "1", + "triggerId": "api_trigger/sample1_API_1", + "startTasks": [ + { + "taskId": "1" + } + ], + "properties": { + "Trigger name": "sample1_API_1" + } + } + ], + "taskConfigs": [ + { + "task": "GenericRestV2Task", + "taskId": "1", + "parameters": { + "additionalHeaders": { + "key": "additionalHeaders", + "value": {} + }, + "authConfigName": { + "key": "authConfigName", + "value": { + "stringValue": "" + } + }, + "disableSSLValidation": { + "key": "disableSSLValidation", + "value": { + "booleanValue": false + } + }, + "followRedirects": { + "key": "followRedirects", + "value": { + "booleanValue": true + } + }, + "httpMethod": { + "key": "httpMethod", + "value": { + "stringValue": "GET" + } + }, + "httpParams": { + "key": "httpParams", + "value": {} + }, + "requestBody": { + "key": "requestBody", + "value": { + "stringValue": "" + } + }, + "requestorId": { + "key": "requestorId", + "value": { + "stringValue": "" + } + }, + "responseBody": { + "key": "responseBody", + "value": { + "stringArray": { + "stringValues": [ + "$`Task_1_responseBody`$" + ] + } + } + }, + "responseHeader": { + "key": "responseHeader", + "value": { + "stringArray": { + "stringValues": [ + "$`Task_1_responseHeader`$" + ] + } + } + }, + "responseStatus": { + "key": "responseStatus", + "value": { + "stringArray": { + "stringValues": [ + "$`Task_1_responseStatus`$" + ] + } + } + }, + "throwError": { + "key": "throwError", + "value": { + "booleanValue": true + } + }, + "url": { + "key": "url", + "value": { + "stringValue": "$`CONFIG_url`$" + } + }, + "urlFetchingService": { + "key": "urlFetchingService", + "value": { + "stringValue": "HARPOON" + } + }, + "urlQueryStrings": { + "key": "urlQueryStrings", + "value": {} + }, + "useSSL": { + "key": "useSSL", + "value": { + "booleanValue": false + } + }, + "userAgent": { + "key": "userAgent", + "value": { + "stringValue": "" + } + } + }, + "displayName": "Call REST Endpoint", + "taskExecutionStrategy": "WHEN_ALL_SUCCEED", + "externalTaskType": "NORMAL_TASK" + } + ], + "integrationParameters": [ + { + "key": "`Task_1_responseHeader`", + "dataType": "STRING_VALUE", + "isTransient": true, + "producer": "1_1" + }, + { + "key": "`Task_1_responseBody`", + "dataType": "STRING_VALUE", + "isTransient": true, + "producer": "1_1" + }, + { + "key": "`Task_1_responseStatus`", + "dataType": "STRING_VALUE", + "isTransient": true, + "producer": "1_1" + } + ], + "integrationConfigParameters": [ + { + "parameter": { + "key": "`CONFIG_url`", + "dataType": "STRING_VALUE", + "defaultValue": { + "stringValue": "https://httpbin.org/get" + }, + "displayName": "`CONFIG_url`" + } + } + ], + "databasePersistencePolicy": "", + "cloudLoggingDetails": { + "cloudLoggingSeverity": "", + "enableCloudLogging": false + } +} diff --git a/samples/upload.json b/samples/upload.json new file mode 100644 index 00000000..9f3a0959 --- /dev/null +++ b/samples/upload.json @@ -0,0 +1,4 @@ +{ + "content": "{\"snapshotNumber\": \"4\",\"triggerConfigs\": [{\"label\": \"API Trigger\",\"startTasks\": [{\"taskId\": \"1\"}],\"properties\": {\"Trigger name\": \"sample_API_1\"},\"triggerType\": \"API\",\"triggerNumber\": \"1\",\"triggerId\": \"api_trigger/sample_API_1\"}],\"taskConfigs\": [{\"task\": \"GenericRestV2Task\",\"taskId\": \"1\",\"parameters\": {\"throwError\": {\"key\": \"throwError\",\"value\": {\"booleanValue\": true}},\"responseBody\": {\"key\": \"responseBody\",\"value\": {\"stringArray\": {\"stringValues\": [\"$`Task_1_responseBody`$\"]}}},\"disableSSLValidation\": {\"key\": \"disableSSLValidation\",\"value\": {\"booleanValue\": false}},\"httpParams\": {\"key\": \"httpParams\"},\"responseHeader\": {\"key\": \"responseHeader\",\"value\": {\"stringArray\": {\"stringValues\": [\"$`Task_1_responseHeader`$\"]}}},\"userAgent\": {\"key\": \"userAgent\",\"value\": {\"stringValue\": \"\"}},\"httpMethod\": {\"key\": \"httpMethod\",\"value\": {\"stringValue\": \"GET\"}},\"responseStatus\": {\"key\": \"responseStatus\",\"value\": {\"stringArray\": {\"stringValues\": [\"$`Task_1_responseStatus`$\"]}}},\"url\": {\"key\": \"url\",\"value\": {\"stringValue\": \"https://httpbin.org/get\"}},\"urlFetchingService\": {\"key\": \"urlFetchingService\",\"value\": {\"stringValue\": \"HARPOON\"}},\"useSSL\": {\"key\": \"useSSL\",\"value\": {\"booleanValue\": false}},\"requestorId\": {\"key\": \"requestorId\",\"value\": {\"stringValue\": \"\"}},\"urlQueryStrings\": {\"key\": \"urlQueryStrings\"},\"requestBody\": {\"key\": \"requestBody\",\"value\": {\"stringValue\": \"\"}},\"followRedirects\": {\"key\": \"followRedirects\",\"value\": {\"booleanValue\": true}},\"additionalHeaders\": {\"key\": \"additionalHeaders\"}},\"taskExecutionStrategy\": \"WHEN_ALL_SUCCEED\",\"displayName\": \"Call httpbin service\"}],\"integrationParameters\": [{\"key\": \"`Task_1_responseHeader`\",\"dataType\": \"STRING_VALUE\",\"displayName\": \"`Task_1_responseHeader`\",\"isTransient\": true,\"producer\": \"1_1\"},{\"key\": \"`Task_1_responseBody`\",\"dataType\": \"STRING_VALUE\",\"displayName\": \"`Task_1_responseBody`\",\"isTransient\": true,\"producer\": \"1_1\"},{\"key\": \"`Task_1_responseStatus`\",\"dataType\": \"STRING_VALUE\",\"displayName\": \"`Task_1_responseStatus`\",\"isTransient\": true,\"producer\": \"1_1\"}],\"cloudLoggingDetails\": {\"cloudLoggingSeverity\": \"INFO\",\"enableCloudLogging\": true},\"databasePersistencePolicy\": \"DATABASE_PERSISTENCE_DISABLED\"}", + "fileFormat": "JSON" +} From f8afeefec74d8f46e8cad120c61aaa6ace9e2950 Mon Sep 17 00:00:00 2001 From: srinandan Date: Sat, 28 Dec 2024 19:03:11 +0000 Subject: [PATCH 4/9] chore: add auth config samples to integrationcli #330 --- docs/docs.go | 8 ++++++++ internal/cmd/authconfigs/authconfigs.go | 11 +++++++++++ internal/cmd/authconfigs/create.go | 4 ++++ internal/cmd/connectors/connectors.go | 8 ++++++++ internal/cmd/connectors/create.go | 1 + internal/cmd/integrations/create.go | 3 ++- internal/cmd/integrations/integrations.go | 1 + samples/ac_authtoken.json | 12 ++++++++++++ samples/ac_oidc.json | 12 ++++++++++++ samples/ac_username.json | 12 ++++++++++++ samples/b64encoded_ac.txt | 8 ++++++++ samples/pub_sub_connection.json | 17 +++++++++++++++++ 12 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 samples/ac_authtoken.json create mode 100644 samples/ac_oidc.json create mode 100644 samples/ac_username.json create mode 100644 samples/b64encoded_ac.txt create mode 100644 samples/pub_sub_connection.json diff --git a/docs/docs.go b/docs/docs.go index 5c531d3b..c53e5f0b 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -22,6 +22,8 @@ import ( "strings" cmd "internal/cmd" + "internal/cmd/authconfigs" + "internal/cmd/connectors" integrations "internal/cmd/integrations" "github.com/spf13/cobra/doc" @@ -42,6 +44,7 @@ Set up integrationcli with preferences: ` + getSingleLine("integrationcli prefs | integrations | ` + getSingleLine(integrations.GetExample(0)) + `| | integrations | ` + getSingleLine(integrations.GetExample(1)) + `| | integrations | ` + getSingleLine(integrations.GetExample(2)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(13)) + `| | integrations | ` + getSingleLine(integrations.GetExample(3)) + `| | integrations | ` + getSingleLine(integrations.GetExample(4)) + `| | integrations | ` + getSingleLine(integrations.GetExample(5)) + `| @@ -52,6 +55,11 @@ Set up integrationcli with preferences: ` + getSingleLine("integrationcli prefs | integrations | ` + getSingleLine(integrations.GetExample(10)) + `| | integrations | ` + getSingleLine(integrations.GetExample(11)) + `| | integrations | ` + getSingleLine(integrations.GetExample(12)) + `| +| authconfigs | ` + getSingleLine(authconfigs.GetExample(0)) + `| +| authconfigs | ` + getSingleLine(authconfigs.GetExample(1)) + `| +| authconfigs | ` + getSingleLine(authconfigs.GetExample(2)) + `| +| authconfigs | ` + getSingleLine(authconfigs.GetExample(3)) + `| +| connectors | ` + getSingleLine(connectors.GetExample(0)) + `| NOTE: This file is auto-generated during a release. Do not modify.` diff --git a/internal/cmd/authconfigs/authconfigs.go b/internal/cmd/authconfigs/authconfigs.go index 78c6a355..f7d64189 100644 --- a/internal/cmd/authconfigs/authconfigs.go +++ b/internal/cmd/authconfigs/authconfigs.go @@ -25,6 +25,13 @@ var Cmd = &cobra.Command{ Long: "Manage integration auth configurations", } +var examples = []string{ + `integrationcli authconfigs create -f samples/ac_username.json`, + `integrationcli authconfigs create -f samples/ac_oidc.json`, + `integrationcli authconfigs create -f samples/ac_authtoken.json`, + `integrationcli authconfigs create -e samples/b64encoded_ac.txt -k locations/$region/keyRings/$key/cryptoKeys/$cryptokey`, +} + func init() { var project, region string @@ -41,3 +48,7 @@ func init() { Cmd.AddCommand(CreateCmd) Cmd.AddCommand(PatchCmd) } + +func GetExample(i int) string { + return examples[i] +} diff --git a/internal/cmd/authconfigs/create.go b/internal/cmd/authconfigs/create.go index 0e02d9ea..b376e473 100644 --- a/internal/cmd/authconfigs/create.go +++ b/internal/cmd/authconfigs/create.go @@ -92,6 +92,10 @@ var CreateCmd = &cobra.Command{ _, err = authconfigs.Create(content) return err }, + Example: `Create a new user name auth config: ` + GetExample(0) + ` +Create a new OIDC auth config: ` + GetExample(1) + ` +Create a new auth token auth config: ` + GetExample(2) + ` +Create a new auth config from Cloud KMS Encrypted files: ` + GetExample(3), } var authConfigFile, encryptedFile, encryptionKey string diff --git a/internal/cmd/connectors/connectors.go b/internal/cmd/connectors/connectors.go index 6e0c3703..f89360e2 100644 --- a/internal/cmd/connectors/connectors.go +++ b/internal/cmd/connectors/connectors.go @@ -25,6 +25,10 @@ var Cmd = &cobra.Command{ Long: "Manage connections for Integration Connectors", } +var examples = []string{ + `integrationcli connectors create -n $name -g=true -f samples/pub_sub_connection.json -sa=connectors --wat=true --default-token`, +} + func init() { var region, project string @@ -48,3 +52,7 @@ func init() { Cmd.AddCommand(CustomCmd) Cmd.AddCommand(EventSubCmd) } + +func GetExample(i int) string { + return examples[i] +} diff --git a/internal/cmd/connectors/create.go b/internal/cmd/connectors/create.go index 150fa3b0..d431b2ca 100644 --- a/internal/cmd/connectors/create.go +++ b/internal/cmd/connectors/create.go @@ -68,6 +68,7 @@ var CreateCmd = &cobra.Command{ return err }, + Example: `Create a PubSub connectors and grant the Service Account permissions: ` + GetExample(0), } var connectionFile, serviceAccountName, serviceAccountProject, encryptionKey string diff --git a/internal/cmd/integrations/create.go b/internal/cmd/integrations/create.go index 4326ac19..0c702b69 100644 --- a/internal/cmd/integrations/create.go +++ b/internal/cmd/integrations/create.go @@ -117,7 +117,8 @@ var CreateCmd = &cobra.Command{ }, Example: `Create a new Inegration Version with a user label: ` + GetExample(0) + ` Create a new Inegration Version with overrides: ` + GetExample(1) + ` -Create a new Inegration Version and publish it: ` + GetExample(2), +Create a new Inegration Version and publish it: ` + GetExample(2) + `, +Create a new Inegration Version and return a basic response: ` + GetExample(13), } var ( diff --git a/internal/cmd/integrations/integrations.go b/internal/cmd/integrations/integrations.go index e981fba5..838757c3 100644 --- a/internal/cmd/integrations/integrations.go +++ b/internal/cmd/integrations/integrations.go @@ -41,6 +41,7 @@ var examples = []string{ `integrationcli integrations apply -f . --env=dev --default-token`, `integrationcli integrations apply -f . --grant-permission=true --default-token`, `integrationcli integrations apply -f . --skip-connectors=true --default-token`, + `integrationcli integrations create -n $name -f samples/sample.json --basic=true --default-token`, } func init() { diff --git a/samples/ac_authtoken.json b/samples/ac_authtoken.json new file mode 100644 index 00000000..f5b7600f --- /dev/null +++ b/samples/ac_authtoken.json @@ -0,0 +1,12 @@ +{ + "displayName": "authconfig-sample", + "description": "this is a sample auth config", + "visibility": "CLIENT_VISIBLE", + "decryptedCredential": { + "credentialType": "AUTH_TOKEN", + "authToken": { + "type": "Bearer", + "token": "test" + } + } +} \ No newline at end of file diff --git a/samples/ac_oidc.json b/samples/ac_oidc.json new file mode 100644 index 00000000..d3e6b76e --- /dev/null +++ b/samples/ac_oidc.json @@ -0,0 +1,12 @@ +{ + "displayName": "authconfig-sample2", + "description": "this is a sample auth config", + "visibility": "CLIENT_VISIBLE", + "decryptedCredential": { + "credentialType": "OIDC_TOKEN", + "oidcToken": { + "serviceAccountEmail": "terraform@nandanks-151422.iam.gserviceaccount.com", + "audience": "https://us-west2-nandanks-151422.cloudfunctions.net/helloWorld" + } + } +} \ No newline at end of file diff --git a/samples/ac_username.json b/samples/ac_username.json new file mode 100644 index 00000000..3f584075 --- /dev/null +++ b/samples/ac_username.json @@ -0,0 +1,12 @@ +{ + "displayName": "authconfig-sample", + "description": "this is a sample auth config", + "visibility": "CLIENT_VISIBLE", + "decryptedCredential": { + "credentialType": "USERNAME_AND_PASSWORD", + "usernameAndPassword": { + "username": "test", + "password": "test" + } + } +} \ No newline at end of file diff --git a/samples/b64encoded_ac.txt b/samples/b64encoded_ac.txt new file mode 100644 index 00000000..275aae14 --- /dev/null +++ b/samples/b64encoded_ac.txt @@ -0,0 +1,8 @@ +CiQANd7GxuDznOZ0qPaXTSleHCOuNOEfeXM70RhK9pq++iisvYMSyQIAmjiXRd83 +x3qeJxNhesaKuBQ2cd7opboDgt6v0M5y3ySOhlGX63mkVzq8sPoLoK/qIjp4VdS/ +hvhDSBEe3pdE1v+uXVESvzM1nCI9R/ztIb5/FfXIoVEmNAkin2KPyCIQrVBrcVxo +AVRHK7OOG5akYtOF+WjHuR7MKr1wVwtgc/o5mHAfR3iPEqg9Lci//T5i3fCEYz7p +dyAN4ZJgOdjNu3H9LCY7Fg4jLgWMkvfL4lMdavXKOBVEaKKHiQksIlLfD2m0zAb3 +xIZip6o8T6ayuxUvRWWksp3PSFfAuEEovXDFXAlJ3o0KPtsXeFIdOc+wLmz0kTks +XNTA7URiK2LxhgYqLXKnco/3UHEx7t1Yhmp6RX839dPH3jaBM0WAfO22zqoI6o7L +R6ojA+jaJnTCiziEviSf5UUIEWtfvYJQaz8hltJRs7aYiQ== diff --git a/samples/pub_sub_connection.json b/samples/pub_sub_connection.json new file mode 100644 index 00000000..821f6f05 --- /dev/null +++ b/samples/pub_sub_connection.json @@ -0,0 +1,17 @@ +{ + "description": "This is a sample", + "connectorDetails": { + "name": "pubsub", + "version": 1 + }, + "configVariables": [ + { + "key": "project_id", + "stringValue": "$PROJECT_ID$" + }, + { + "key": "topic_id", + "stringValue": "mytopic" + } + ] +} From ebc7efd1c0714d9adb0650462d1363509a7741a5 Mon Sep 17 00:00:00 2001 From: srinandan Date: Sat, 28 Dec 2024 19:05:23 +0000 Subject: [PATCH 5/9] chore: gofumpt files #330 --- docs/docs.go | 5 +++-- internal/cmd/integrations/apply.go | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index c53e5f0b..74e6d139 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -15,6 +15,8 @@ package main import ( + "internal/cmd/authconfigs" + "internal/cmd/connectors" "io" "log" "os" @@ -22,8 +24,7 @@ import ( "strings" cmd "internal/cmd" - "internal/cmd/authconfigs" - "internal/cmd/connectors" + integrations "internal/cmd/integrations" "github.com/spf13/cobra/doc" diff --git a/internal/cmd/integrations/apply.go b/internal/cmd/integrations/apply.go index 133a5e4a..fbb78c2c 100644 --- a/internal/cmd/integrations/apply.go +++ b/internal/cmd/integrations/apply.go @@ -646,8 +646,7 @@ func processCodeFolders(javascriptFolder string, jsonnetFolder string) (codeMap if err != nil { return nil, err } - codeMap["JavaScriptTask"][strings.ReplaceAll(getFilenameWithoutExtension(javascriptName), "javascript_", "")] = - strings.ReplaceAll(string(javascriptBytes), "\n", "\\n") + codeMap["JavaScriptTask"][strings.ReplaceAll(getFilenameWithoutExtension(javascriptName), "javascript_", "")] = strings.ReplaceAll(string(javascriptBytes), "\n", "\\n") } } @@ -671,8 +670,7 @@ func processCodeFolders(javascriptFolder string, jsonnetFolder string) (codeMap if err != nil { return nil, err } - codeMap["JsonnetMapperTask"][strings.ReplaceAll(getFilenameWithoutExtension(jsonnetName), "datatransformer_", "")] = - strings.ReplaceAll(string(jsonnetBytes), "\n", "\\n") + codeMap["JsonnetMapperTask"][strings.ReplaceAll(getFilenameWithoutExtension(jsonnetName), "datatransformer_", "")] = strings.ReplaceAll(string(jsonnetBytes), "\n", "\\n") } } From 5872c8f0a21bc6a9347e67e8dcc56d26f187565c Mon Sep 17 00:00:00 2001 From: srinandan Date: Sat, 28 Dec 2024 19:16:03 +0000 Subject: [PATCH 6/9] chore: update dependencies #330 --- go.mod | 4 ++-- go.sum | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1216d3a1..a6e56a7a 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/GoogleCloudPlatform/application-integration-management-toolkit go 1.23.2 -require internal/apiclient v1.0.0 // indirect +require internal/apiclient v1.0.0 replace internal/apiclient => ./internal/apiclient @@ -80,7 +80,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect go.opentelemetry.io/otel/trace v1.31.0 // indirect golang.org/x/crypto v0.31.0 // indirect - golang.org/x/net v0.31.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/go.sum b/go.sum index f2a5ccc7..7fb3b2be 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= From 4b764de26d85652eb8c684b7020990b0b1fea5c8 Mon Sep 17 00:00:00 2001 From: srinandan Date: Sun, 29 Dec 2024 03:57:38 +0000 Subject: [PATCH 7/9] chore: add samples for custom connectors #330 --- internal/cmd/connectors/connectors.go | 32 ++++++++++++++++++++++++- internal/cmd/connectors/create.go | 3 ++- internal/cmd/connectors/crtcustom.go | 8 ++++--- internal/cmd/connectors/crtcustomver.go | 1 + internal/cmd/integrations/scaffold.go | 2 +- 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/internal/cmd/connectors/connectors.go b/internal/cmd/connectors/connectors.go index f89360e2..19f2c9bd 100644 --- a/internal/cmd/connectors/connectors.go +++ b/internal/cmd/connectors/connectors.go @@ -15,6 +15,8 @@ package connectors import ( + "fmt" + "github.com/spf13/cobra" ) @@ -26,9 +28,19 @@ var Cmd = &cobra.Command{ } var examples = []string{ - `integrationcli connectors create -n $name -g=true -f samples/pub_sub_connection.json -sa=connectors --wat=true --default-token`, + `integrationcli connectors create -n $name -g=true -f samples/pub_sub_connection.json -sa=connectors --wait=true --default-token`, + `integrationcli connectors create -n $name -f samples/gcs_connection.json -sa=connectors --wait=true --default-token`, + `integrationcli connectors custom versions create --id $version -n $name -f samples/custom-connection.json --sa=connectors --default-token`, + `integrationcli connectors custom create -n $name -d $dispName --type OPEN_API --default-token`, } +type ConnectorType string + +const ( + OPEN_API ConnectorType = "OPEN_API" + PROTO ConnectorType = "PROTO" +) + func init() { var region, project string @@ -56,3 +68,21 @@ func init() { func GetExample(i int) string { return examples[i] } + +func (c *ConnectorType) String() string { + return string(*c) +} + +func (c *ConnectorType) Set(r string) error { + switch r { + case "OPEN_API", "PROTO": + *c = ConnectorType(r) + default: + return fmt.Errorf("must be %s or %s", OPEN_API, PROTO) + } + return nil +} + +func (c *ConnectorType) Type() string { + return "ConnectorType" +} diff --git a/internal/cmd/connectors/create.go b/internal/cmd/connectors/create.go index d431b2ca..11f21a90 100644 --- a/internal/cmd/connectors/create.go +++ b/internal/cmd/connectors/create.go @@ -68,7 +68,8 @@ var CreateCmd = &cobra.Command{ return err }, - Example: `Create a PubSub connectors and grant the Service Account permissions: ` + GetExample(0), + Example: `Create a PubSub connector and grant the Service Account permissions: ` + GetExample(0) + ` +Create a GCS Connector: ` + GetExample(1), } var connectionFile, serviceAccountName, serviceAccountProject, encryptionKey string diff --git a/internal/cmd/connectors/crtcustom.go b/internal/cmd/connectors/crtcustom.go index ded1ae70..fbfaff78 100644 --- a/internal/cmd/connectors/crtcustom.go +++ b/internal/cmd/connectors/crtcustom.go @@ -50,12 +50,14 @@ var CrtCustomCmd = &cobra.Command{ return err }, + Example: `Create a custom connector for OPEN_API type: ` + GetExample(3), } var labels map[string]string +var connType ConnectorType func init() { - var name, description, displayName, connType string + var name, description, displayName string CrtCustomCmd.Flags().StringVarP(&name, "name", "n", "", "Connection name") @@ -63,8 +65,8 @@ func init() { "", "Custom Connection display name") CrtCustomCmd.Flags().StringVarP(&description, "description", "", "", "Custom Connection description") - CrtCustomCmd.Flags().StringVarP(&connType, "type", "", - "", "Custom Connection type") + CrtCustomCmd.Flags().Var(&connType, "type", + "Custom Connection type must be set to OPEN_API or PROTO") CrtCustomCmd.Flags().StringToStringVarP(&labels, "labels", "l", map[string]string{}, "Custom Connection labels") diff --git a/internal/cmd/connectors/crtcustomver.go b/internal/cmd/connectors/crtcustomver.go index 54ceb26e..b1bb2e3f 100644 --- a/internal/cmd/connectors/crtcustomver.go +++ b/internal/cmd/connectors/crtcustomver.go @@ -53,6 +53,7 @@ var CrtCustomVerCmd = &cobra.Command{ _, err = connections.CreateCustomVersion(name, id, content, serviceAccountName, serviceAccountProject) return err }, + Example: `Create a custom connection version: ` + GetExample(2), } func init() { diff --git a/internal/cmd/integrations/scaffold.go b/internal/cmd/integrations/scaffold.go index a6e567d1..f4c5469a 100644 --- a/internal/cmd/integrations/scaffold.go +++ b/internal/cmd/integrations/scaffold.go @@ -369,7 +369,7 @@ var ScaffoldCmd = &cobra.Command{ Example: `Generate scaffold for dev env using snapshot: ` + GetExample(5) + ` Generate scaffold for integration, but skip connectors: ` + GetExample(6) + ` Generate scaffold for integration and produce cloud build config: ` + GetExample(7) + ` -Generate scaffold for integration and produce cloud deploy config: ` + GetExample(8), +Generate scaffold for integration and produce cloud deploy config: ` + GetExample(8) + `\n See samples/scaffold-sample for more details`, } var ( From 693fb5a347502032fc049ea4695130dc0e18fcd0 Mon Sep 17 00:00:00 2001 From: srinandan Date: Sun, 29 Dec 2024 06:55:33 +0000 Subject: [PATCH 8/9] feat: adds ability to publish and unpublish latest ver #330 --- internal/cmd/integrations/publish.go | 66 ++++++++++++++++++++++---- internal/cmd/integrations/unpublish.go | 26 +++++++++- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/internal/cmd/integrations/publish.go b/internal/cmd/integrations/publish.go index 03f444a6..91514f05 100644 --- a/internal/cmd/integrations/publish.go +++ b/internal/cmd/integrations/publish.go @@ -15,9 +15,12 @@ package integrations import ( + "encoding/json" "errors" + "fmt" "internal/apiclient" "internal/client/integrations" + "internal/clilog" "os" "github.com/spf13/cobra" @@ -47,6 +50,7 @@ var PublishVerCmd = &cobra.Command{ configVarsJson := cmd.Flag("config-vars-json").Value.String() configVarsFile := cmd.Flag("config-vars").Value.String() var contents []byte + var info string if configVarsJson == "" { if configVarsFile != "" { @@ -63,17 +67,40 @@ var PublishVerCmd = &cobra.Command{ contents = []byte(configVarsJson) } - if version != "" { + if latest { + apiclient.DisableCmdPrintHttpResponse() + // list integration versions, order by state=SNAPSHOT, page size = 1 and return basic info + respBody, err := integrations.ListVersions(name, 1, "", "state=SNAPSHOT", + "snapshot_number", false, false, true) + if err != nil { + return fmt.Errorf("unable to list versions: %v", err) + } + version, err = getIntegrationVersion(respBody) + if err != nil { + return err + } + apiclient.EnableCmdPrintHttpResponse() + _, err = integrations.Publish(name, version, contents) + info = "version " + version + } else if version != "" { _, err = integrations.Publish(name, version, contents) + info = "version " + version } else if userLabel != "" { _, err = integrations.PublishUserLabel(name, userLabel, contents) + info = "user label " + userLabel } else if snapshot != "" { _, err = integrations.PublishSnapshot(name, snapshot, contents) + info = "snapshot number " + snapshot + } + if err == nil { + clilog.Info.Printf("Integration %s %s published successfully\n", name, info) } return err }, } +var latest bool + func init() { var name, version, configVars, configVarsJson string @@ -89,20 +116,41 @@ func init() { "", "Path to file containing config variables") PublishVerCmd.Flags().StringVarP(&configVarsJson, "config-vars-json", "", "", "Json string containing the config variables if both Json string and file is present Json string will only be used.") + PublishVerCmd.Flags().BoolVarP(&latest, "latest", "", + true, "Publishes the integeration version with the highest snapshot number in SNAPSHOT state; default is true") _ = PublishVerCmd.MarkFlagRequired("name") } func validate(version string) (err error) { switch { - case version == "" && userLabel == "" && snapshot == "": - return errors.New("must pass oneOf version, snapshot or user-label") - case version != "" && (userLabel != "" || snapshot != ""): - return errors.New("must pass oneOf version, snapshot or user-label") - case userLabel != "" && (version != "" || snapshot != ""): - return errors.New("must pass oneOf version, snapshot or user-label") - case snapshot != "" && (userLabel != "" || version != ""): - return errors.New("must pass oneOf version, snapshot or user-label") + case !latest && (version == "" && userLabel == "" && snapshot == ""): + return errors.New("must pass oneOf version, snapshot or user-label or set latest to true") + case !latest && (version != "" && (userLabel != "" || snapshot != "")): + return errors.New("must pass oneOf version, snapshot or user-label or set latest to true") + case !latest && (userLabel != "" && (version != "" || snapshot != "")): + return errors.New("must pass oneOf version, snapshot or user-label or set latest to true") + case !latest && (snapshot != "" && (userLabel != "" || version != "")): + return errors.New("must pass oneOf version, snapshot or user-label or set latest to true") + case latest && (version != "" || userLabel != "" || snapshot != ""): + return errors.New("latest cannot be combined with version, snapshot or user-label") } return nil } + +func getIntegrationVersion(respBody []byte) (string, error) { + var data map[string]interface{} + err := json.Unmarshal(respBody, &data) + if err != nil { + return "", err + } + if data["integrationVersions"] == nil { + return "", fmt.Errorf("no integration versions were found") + } + integrationVersions := data["integrationVersions"].([]interface{}) + firstIntegrationVersion := integrationVersions[0].(map[string]interface{}) + if firstIntegrationVersion["version"].(string) == "" { + return "", fmt.Errorf("unable to extract version id from integration") + } + return firstIntegrationVersion["version"].(string), nil +} diff --git a/internal/cmd/integrations/unpublish.go b/internal/cmd/integrations/unpublish.go index 36946814..bee9c234 100644 --- a/internal/cmd/integrations/unpublish.go +++ b/internal/cmd/integrations/unpublish.go @@ -15,8 +15,10 @@ package integrations import ( + "fmt" "internal/apiclient" "internal/client/integrations" + "internal/clilog" "github.com/spf13/cobra" ) @@ -43,13 +45,33 @@ var UnPublishVerCmd = &cobra.Command{ version := cmd.Flag("ver").Value.String() name := cmd.Flag("name").Value.String() - if version != "" { + var info string + + if latest { + apiclient.DisableCmdPrintHttpResponse() + // list integration versions, order by state=ACTIVE, page size = 1 and return basic info + respBody, err := integrations.ListVersions(name, 1, "", "state=ACTIVE", + "snapshot_number", false, false, true) + if err != nil { + return fmt.Errorf("unable to list versions: %v", err) + } + version, err = getIntegrationVersion(respBody) + if err != nil { + return err + } + apiclient.EnableCmdPrintHttpResponse() + _, err = integrations.Unpublish(name, version) + info = "version " + version + } else if version != "" { _, err = integrations.Unpublish(name, version) } else if userLabel != "" { _, err = integrations.UnpublishUserLabel(name, userLabel) } else if snapshot != "" { _, err = integrations.UnpublishSnapshot(name, snapshot) } + if err == nil { + clilog.Info.Printf("Integration %s %s unpublished successfully\n", name, info) + } return err }, } @@ -65,6 +87,8 @@ func init() { "", "Integration flow user label") UnPublishVerCmd.Flags().StringVarP(&snapshot, "snapshot", "s", "", "Integration flow snapshot number") + UnPublishVerCmd.Flags().BoolVarP(&latest, "latest", "", + true, "Unpublishes the integeration version with the highest snapshot number in SNAPSHOT state; default is true") _ = UnPublishVerCmd.MarkFlagRequired("name") } From 9be4e5786154ef128165b9f9bcb31406c3d74e27 Mon Sep 17 00:00:00 2001 From: srinandan Date: Sun, 29 Dec 2024 07:04:59 +0000 Subject: [PATCH 9/9] chore: adds samples for publish and unpublish #330 --- docs/docs.go | 4 ++++ internal/cmd/integrations/integrations.go | 4 ++++ internal/cmd/integrations/publish.go | 2 ++ internal/cmd/integrations/unpublish.go | 2 ++ 4 files changed, 12 insertions(+) diff --git a/docs/docs.go b/docs/docs.go index 74e6d139..32669b77 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -56,6 +56,10 @@ Set up integrationcli with preferences: ` + getSingleLine("integrationcli prefs | integrations | ` + getSingleLine(integrations.GetExample(10)) + `| | integrations | ` + getSingleLine(integrations.GetExample(11)) + `| | integrations | ` + getSingleLine(integrations.GetExample(12)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(14)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(15)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(16)) + `| +| integrations | ` + getSingleLine(integrations.GetExample(17)) + `| | authconfigs | ` + getSingleLine(authconfigs.GetExample(0)) + `| | authconfigs | ` + getSingleLine(authconfigs.GetExample(1)) + `| | authconfigs | ` + getSingleLine(authconfigs.GetExample(2)) + `| diff --git a/internal/cmd/integrations/integrations.go b/internal/cmd/integrations/integrations.go index 838757c3..bf507934 100644 --- a/internal/cmd/integrations/integrations.go +++ b/internal/cmd/integrations/integrations.go @@ -42,6 +42,10 @@ var examples = []string{ `integrationcli integrations apply -f . --grant-permission=true --default-token`, `integrationcli integrations apply -f . --skip-connectors=true --default-token`, `integrationcli integrations create -n $name -f samples/sample.json --basic=true --default-token`, + `integrationcli integrations versions publish -n $name --default-token`, + `integrationcli integrations versions publish -n $name -s $snapshot --default-token`, + `integrationcli integrations versions unpublish -n $name --default-token`, + `integrationcli integrations versions unpublish -n $name -u $userLabel --default-token`, } func init() { diff --git a/internal/cmd/integrations/publish.go b/internal/cmd/integrations/publish.go index 91514f05..e92f153c 100644 --- a/internal/cmd/integrations/publish.go +++ b/internal/cmd/integrations/publish.go @@ -97,6 +97,8 @@ var PublishVerCmd = &cobra.Command{ } return err }, + Example: `Publishes an integration vesion with the highest snapshot in SNAPHOST state: ` + GetExample(14) + ` +Publishes an integration version that matches user supplied snapshot number: ` + GetExample(15), } var latest bool diff --git a/internal/cmd/integrations/unpublish.go b/internal/cmd/integrations/unpublish.go index bee9c234..ace0e870 100644 --- a/internal/cmd/integrations/unpublish.go +++ b/internal/cmd/integrations/unpublish.go @@ -74,6 +74,8 @@ var UnPublishVerCmd = &cobra.Command{ } return err }, + Example: `Unpublishes an integration vesion with the highest snapshot in SNAPHOST state: ` + GetExample(16) + ` +Unpublishes an integration version that matches user supplied user label: ` + GetExample(17), } func init() {