Merge pull request #23 from harness/CI-10795

fix: [CI-10795]: support secret type for output varaible

api/api.go

@@ -93,10 +93,11 @@ type (
 		Run        RunConfig         `json:"run,omitempty"`
 		RunTest    RunTestConfig     `json:"run_test,omitempty"`
 
-		OutputVars        []string   `json:"output_vars,omitempty"`
-		TestReport        TestReport `json:"test_report,omitempty"`
-		Timeout           int        `json:"timeout,omitempty"` // step timeout in seconds
-		MountDockerSocket *bool      `json:"mount_docker_socket"`
+		OutputVars        []string    `json:"output_vars,omitempty"`
+		TestReport        TestReport  `json:"test_report,omitempty"`
+		Timeout           int         `json:"timeout,omitempty"` // step timeout in seconds
+		MountDockerSocket *bool       `json:"mount_docker_socket"`
+		Outputs           []*OutputV2 `json:"outputs,omitempty"`
 
 		// Valid only for steps running on docker container
 		Auth         *spec.Auth           `json:"auth,omitempty"`
@@ -125,6 +126,12 @@ type (
 		Files        []*spec.File         `json:"files,omitempty"`
 	}
 
+	OutputV2 struct {
+		Key   string `json:"key,omitempty"`
+		Value string `json:"value,omitempty"`
+		Type  string `json:"type,omitempty"`
+	}
+
 	DelegateMetaInfo struct {
 		ID       string `json:"id"`
 		HostName string `json:"host_name"`
@@ -159,6 +166,7 @@ type (
 		OOMKilled bool              `json:"oom_killed,omitempty"`
 		Outputs   map[string]string `json:"outputs,omitempty"`
 		Artifact  []byte            `json:"artifact,omitempty"`
+		OutputV2  []*OutputV2       `json:"outputV2,omitempty"`
 	}
 
 	StreamOutputRequest struct {

executor/executor.go

@@ -14,7 +14,7 @@ var (
 	once     sync.Once
 )
 
-//TODO:xun add mutex
+// TODO:xun add mutex
 // Executor maps stage runtime ID to the state of the stage
 type Executor struct {
 	m  map[string]*StageData

pipeline/runtime/common.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/harness/harness-docker-runner/api"
 	"github.com/harness/harness-docker-runner/logstream"
 	"github.com/sirupsen/logrus"
 )
@@ -51,6 +52,29 @@ func getOutputVarCmd(entrypoint, outputVars []string, outputFile string) string
 	return cmd
 }
 
+func getOutputsCmd(entrypoint []string, outputVars []*api.OutputV2, outputFile string) string {
+	isPsh := isPowershell(entrypoint)
+	isPython := isPython(entrypoint)
+
+	cmd := ""
+	if isPsh {
+		cmd += fmt.Sprintf("\nNew-Item %s", outputFile)
+	} else if isPython {
+		cmd += "\nimport os\n"
+	}
+	for _, o := range outputVars {
+		if isPsh {
+			cmd += fmt.Sprintf("\n$val = \"%s $Env:%s\" \nAdd-Content -Path %s -Value $val", o.Key, o.Value, outputFile)
+		} else if isPython {
+			cmd += fmt.Sprintf("with open('%s', 'a') as out_file:\n\tout_file.write('%s ' + os.getenv('%s') + '\\n')\n", outputFile, o.Key, o.Value)
+		} else {
+			cmd += fmt.Sprintf("\necho \"%s $%s\" >> %s", o.Key, o.Value, outputFile)
+		}
+	}
+
+	return cmd
+}
+
 func fetchArtifactDataFromArtifactFile(artifactFile string, out io.Writer) ([]byte, error) {
 	log := logrus.New()
 	log.Out = out

pipeline/runtime/run.go

@@ -22,19 +22,21 @@ import (
 )
 
 func executeRunStep(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer, tiConfig *tiCfg.Cfg) (
-	*runtime.State, map[string]string, []byte, error) {
+	*runtime.State, map[string]string, []byte, []*api.OutputV2, error) {
 	step := toStep(r)
 	step.Command = r.Run.Command
 	step.Entrypoint = r.Run.Entrypoint
 
-	if len(r.OutputVars) > 0 && (len(step.Entrypoint) == 0 || len(step.Command) == 0) {
-		return nil, nil, nil, fmt.Errorf("output variable should not be set for unset entrypoint or command")
+	if (len(r.OutputVars) > 0 || len(r.Outputs) > 0) && (len(step.Entrypoint) == 0 || len(step.Command) == 0) {
+		return nil, nil, nil, nil, fmt.Errorf("output variable should not be set for unset entrypoint or command")
 	}
 
 	outputFile := fmt.Sprintf("%s/%s.out", pipeline.SharedVolPath, step.ID)
 	step.Envs["DRONE_OUTPUT"] = outputFile
 
-	if len(r.OutputVars) > 0 {
+	if len(r.Outputs) > 0 {
+		step.Command[0] += getOutputsCmd(step.Entrypoint, r.Outputs, outputFile)
+	} else if len(r.OutputVars) > 0 {
 		step.Command[0] += getOutputVarCmd(step.Entrypoint, r.OutputVars, outputFile)
 	}
 
@@ -56,14 +58,27 @@ func executeRunStep(ctx context.Context, engine *engine.Engine, r *api.StartStep
 	if exited != nil && exited.Exited && exited.ExitCode == 0 {
 		outputs, err := fetchOutputVariables(outputFile, out) // nolint:govet
 		if err != nil {
-			return exited, nil, nil, err
+			return exited, nil, nil, nil, err
 		}
 		// Delete output variable file
 		if ferr := os.Remove(outputFile); ferr != nil {
 			logrus.WithError(ferr).WithField("file", outputFile).Warnln("could not remove output file")
 		}
-		return exited, outputs, artifact, err
+		if len(r.Outputs) > 0 {
+			outputsV2 := []*api.OutputV2{}
+			for _, output := range r.Outputs {
+				if _, ok := outputs[output.Key]; ok {
+					outputsV2 = append(outputsV2, &api.OutputV2{
+						Key:   output.Key,
+						Value: outputs[output.Key],
+						Type:  output.Type,
+					})
+				}
+			}
+			return exited, outputs, artifact, outputsV2, err
+		}
+		return exited, outputs, artifact, nil, err
 	}
 
-	return exited, nil, artifact, err
+	return exited, nil, artifact, nil, err
 }

pipeline/runtime/runtest.go

@@ -22,26 +22,28 @@ import (
 )
 
 func executeRunTestStep(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer, tiConfig *tiCfg.Cfg) (
-	*runtime.State, map[string]string, []byte, error) {
+	*runtime.State, map[string]string, []byte, []*api.OutputV2, error) {
 	log := logrus.New()
 	log.Out = out
 
 	start := time.Now()
 	cmd, err := instrumentation.GetCmd(ctx, &r.RunTest, r.Name, r.WorkingDir, log, r.Envs, tiConfig)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, nil, nil, err
 	}
 
 	step := toStep(r)
 	step.Command = []string{cmd}
 	step.Entrypoint = r.RunTest.Entrypoint
 
-	if len(r.OutputVars) > 0 && len(step.Entrypoint) == 0 || len(step.Command) == 0 {
-		return nil, nil, nil, fmt.Errorf("output variable should not be set for unset entrypoint or command")
+	if (len(r.OutputVars) > 0 || len(r.Outputs) > 0) && len(step.Entrypoint) == 0 || len(step.Command) == 0 {
+		return nil, nil, nil, nil, fmt.Errorf("output variable should not be set for unset entrypoint or command")
 	}
 
 	outputFile := fmt.Sprintf("%s/%s.out", pipeline.SharedVolPath, step.ID)
-	if len(r.OutputVars) > 0 {
+	if len(r.Outputs) > 0 {
+		step.Command[0] += getOutputsCmd(step.Entrypoint, r.Outputs, outputFile)
+	} else if len(r.OutputVars) > 0 {
 		step.Command[0] += getOutputVarCmd(step.Entrypoint, r.OutputVars, outputFile)
 	}
 
@@ -58,15 +60,26 @@ func executeRunTestStep(ctx context.Context, engine *engine.Engine, r *api.Start
 	}
 
 	artifact, _ := fetchArtifactDataFromArtifactFile(artifactFile, out)
-	if len(r.OutputVars) > 0 {
+	if len(r.Outputs) > 0 {
 		if exited != nil && exited.Exited && exited.ExitCode == 0 {
 			outputs, err := fetchOutputVariables(outputFile, out) // nolint:govet
-			if err != nil {
-				return exited, nil, nil, err
+			outputsV2 := []*api.OutputV2{}
+			for _, output := range r.Outputs {
+				if _, ok := outputs[output.Key]; ok {
+					outputsV2 = append(outputsV2, &api.OutputV2{
+						Key:   output.Key,
+						Value: outputs[output.Key],
+						Type:  output.Type,
+					})
+				}
 			}
-			return exited, outputs, artifact, err
+			return exited, outputs, artifact, outputsV2, err
+		}
+	} else if len(r.OutputVars) > 0 {
+		if exited != nil && exited.Exited && exited.ExitCode == 0 {
+			outputs, err := fetchOutputVariables(outputFile, out) //nolint:govet
+			return exited, outputs, artifact, nil, err
 		}
 	}
-
-	return exited, nil, artifact, err
+	return exited, nil, artifact, nil, err
 }

pipeline/runtime/step_executor.go

@@ -31,6 +31,7 @@ type StepStatus struct {
 	StepErr  error
 	Outputs  map[string]string
 	Artifact []byte
+	OutputV2 []*api.OutputV2
 }
 
 const (
@@ -73,8 +74,8 @@ func (e *StepExecutor) StartStep(ctx context.Context, r *api.StartStepRequest, s
 	e.mu.Unlock()
 
 	go func() {
-		state, outputs, artifact, stepErr := e.executeStep(r, secrets, client, tiConfig)
-		status := StepStatus{Status: Complete, State: state, StepErr: stepErr, Outputs: outputs, Artifact: artifact}
+		state, outputs, artifact, outputV2, stepErr := e.executeStep(r, secrets, client, tiConfig)
+		status := StepStatus{Status: Complete, State: state, StepErr: stepErr, Outputs: outputs, Artifact: artifact, OutputV2: outputV2}
 		e.mu.Lock()
 		e.stepStatus[r.ID] = status
 		channels := e.stepWaitCh[r.ID]
@@ -198,7 +199,7 @@ func (e *StepExecutor) executeStepDrone(r *api.StartStepRequest, tiConfig *tiCfg
 
 		r.Kind = api.Run // only this kind is supported
 
-		exited, _, _, err := e.run(ctx, e.engine, r, stepLog, tiConfig)
+		exited, _, _, _, err := e.run(ctx, e.engine, r, stepLog, tiConfig)
 		if ctx.Err() == context.Canceled || ctx.Err() == context.DeadlineExceeded {
 			logr.WithError(err).Warnln("step execution canceled")
 			return nil, ctx.Err()
@@ -229,10 +230,10 @@ func (e *StepExecutor) executeStepDrone(r *api.StartStepRequest, tiConfig *tiCfg
 	return runStep()
 }
 
-func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, client logstream.Client, tiConfig *tiCfg.Cfg) (*runtime.State, map[string]string, []byte, error) {
+func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, client logstream.Client, tiConfig *tiCfg.Cfg) (*runtime.State, map[string]string, []byte, []*api.OutputV2, error) {
 	if r.LogDrone {
 		state, err := e.executeStepDrone(r, tiConfig)
-		return state, nil, nil, err
+		return state, nil, nil, nil, err
 	}
 
 	wc := livelog.New(client, r.LogKey, r.Name, getNudges())
@@ -255,7 +256,7 @@ func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, cl
 			e.run(ctx, e.engine, r, wr, tiConfig) // nolint:errcheck
 			wc.Close()
 		}()
-		return &runtime.State{Exited: false}, nil, nil, nil
+		return &runtime.State{Exited: false}, nil, nil, nil, nil
 	}
 
 	var result error
@@ -267,7 +268,7 @@ func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, cl
 		defer cancel()
 	}
 
-	exited, outputs, artifact, err := e.run(ctx, e.engine, r, wr, tiConfig)
+	exited, outputs, artifact, outputV2, err := e.run(ctx, e.engine, r, wr, tiConfig)
 	if err != nil {
 		result = multierror.Append(result, err)
 	}
@@ -282,7 +283,7 @@ func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, cl
 	// DeadlineExceeded error this indicates the step was timed out.
 	switch ctx.Err() {
 	case context.Canceled, context.DeadlineExceeded:
-		return nil, nil, nil, ctx.Err()
+		return nil, nil, nil, nil, ctx.Err()
 	}
 
 	if exited != nil {
@@ -298,11 +299,11 @@ func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, cl
 			logrus.WithField("id", r.ID).Infof("received exit code %d\n", exited.ExitCode)
 		}
 	}
-	return exited, outputs, artifact, result
+	return exited, outputs, artifact, outputV2, result
 }
 
 func (e *StepExecutor) run(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer, tiConfig *tiCfg.Cfg) (
-	*runtime.State, map[string]string, []byte, error) {
+	*runtime.State, map[string]string, []byte, []*api.OutputV2, error) {
 	if r.Kind == api.Run {
 		return executeRunStep(ctx, engine, r, out, tiConfig)
 	}
@@ -314,6 +315,7 @@ func convertStatus(status StepStatus) *api.PollStepResponse {
 		Exited:   true,
 		Outputs:  status.Outputs,
 		Artifact: status.Artifact,
+		OutputV2: status.OutputV2,
 	}
 
 	stepErr := status.StepErr

util/util.go

@@ -12,7 +12,7 @@ import (
 
 var onlyOnce sync.Once
 
-// Register capacity of docker runner node 
+// Register capacity of docker runner node
 func RegisterDelegateCapacity(Id string) {
 	onlyOnce.Do(func() {
 		c := config.GetConfig()