Skip to content

Commit

Permalink
(fix) Creation of Multiple ExperimentStatuses fixed (#60)
Browse files Browse the repository at this point in the history
* (fix) ExperimentStatuses Fixed

Signed-off-by: Rahul M Chheda <rahul.chheda@mayadata.io>
  • Loading branch information
rahulchheda authored Apr 8, 2020
1 parent 2d74647 commit 53bcf3b
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 28 deletions.
38 changes: 17 additions & 21 deletions bin/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,71 +19,67 @@ func main() {
// Fetching all the ENV's needed
utils.GetOsEnv(&engineDetails)
klog.V(0).Infoln("Experiments List: ", engineDetails.Experiments, " ", "Engine Name: ", engineDetails.Name, " ", "appLabels : ", engineDetails.AppLabel, " ", "appKind: ", engineDetails.AppKind, " ", "Service Account Name: ", engineDetails.SvcAccount, "Engine Namespace: ", engineDetails.EngineNamespace)

experimentList := utils.CreateExperimentList(&engineDetails)
if err := utils.InitialPatchEngine(engineDetails, clients, experimentList); err != nil {
klog.Errorf("Unable to create Initial ExpeirmentStatus in ChaosEngine, due to error: %v", err)
}
recorder, err := utils.NewEventRecorder(clients, engineDetails)
if err != nil {
klog.Errorf("Unable to initiate EventRecorder for Chaos-Runner, would not be able to add events")
}
// Steps for each Experiment
for i := range engineDetails.Experiments {
for _, experiment := range experimentList {

// Sending event to GA instance
if engineDetails.ClientUUID != "" {
analytics.TriggerAnalytics(engineDetails.Experiments[i], engineDetails.ClientUUID)
analytics.TriggerAnalytics(experiment.Name, engineDetails.ClientUUID)
}

experiment := utils.NewExperimentDetails(&engineDetails, i)

if err := experiment.SetValueFromChaosResources(&engineDetails, clients); err != nil {
klog.V(0).Infof("Unable to set values from Chaos Resources due to error: %v", err)
recorder.ExperimentSkipped(engineDetails.Experiments[i], utils.ExperimentNotFoundErrorReason)
recorder.ExperimentSkipped(experiment.Name, utils.ExperimentNotFoundErrorReason)
}

if err := experiment.SetENV(engineDetails, clients); err != nil {
klog.V(0).Infof("Unable to patch ENV due to error: %v", err)
recorder.ExperimentSkipped(engineDetails.Experiments[i], utils.ExperimentEnvParseErrorReason)
recorder.ExperimentSkipped(experiment.Name, utils.ExperimentEnvParseErrorReason)
break
}
experimentStatus := utils.ExperimentStatus{}
experimentStatus.InitialExperimentStatus(experiment)
if err := experimentStatus.InitialPatchEngine(engineDetails, clients); err != nil {
klog.V(0).Infof("Unable to set Initial Status in ChaosEngine, due to error: %v", err)
}

klog.V(0).Infof("Preparing to run Chaos Experiment: %v", experiment.Name)

if err := experiment.PatchResources(engineDetails, clients); err != nil {
klog.V(0).Infof("Unable to patch Chaos Resources required for Chaos Experiment: %v, due to error: %v", experiment.Name, err)
}

recorder.ExperimentDepedencyCheck(engineDetails.Experiments[i])
recorder.ExperimentDepedencyCheck(experiment.Name)

// Creation of PodTemplateSpec, and Final Job
if err := utils.BuildingAndLaunchJob(experiment, clients); err != nil {
if err := utils.BuildingAndLaunchJob(&experiment, clients); err != nil {
klog.V(0).Infof("Unable to construct chaos experiment job due to: %v", err)
recorder.ExperimentSkipped(engineDetails.Experiments[i], utils.ExperimentJobCreationErrorReason)
recorder.ExperimentSkipped(experiment.Name, utils.ExperimentJobCreationErrorReason)
break
}
recorder.ExperimentJobCreate(engineDetails.Experiments[i], experiment.JobName)
recorder.ExperimentJobCreate(experiment.Name, experiment.JobName)

klog.V(0).Infof("Started Chaos Experiment Name: %v, with Job Name: %v", experiment.Name, experiment.JobName)
// Watching the Job till Completion
if err := engineDetails.WatchJobForCompletion(experiment, clients); err != nil {
if err := engineDetails.WatchJobForCompletion(&experiment, clients); err != nil {
klog.V(0).Infof("Unable to Watch the Job, error: %v", err)
recorder.ExperimentSkipped(engineDetails.Experiments[i], utils.ExperimentJobWatchErrorReason)
recorder.ExperimentSkipped(experiment.Name, utils.ExperimentJobWatchErrorReason)
break
}

// Will Update the chaosEngine Status
if err := engineDetails.UpdateEngineWithResult(experiment, clients); err != nil {
if err := engineDetails.UpdateEngineWithResult(&experiment, clients); err != nil {
klog.V(0).Infof("Unable to Update ChaosEngine Status due to: %v", err)
}

// Delete / retain the Job, using the jobCleanUpPolicy
jobCleanUpPolicy, err := engineDetails.DeleteJobAccordingToJobCleanUpPolicy(experiment, clients)
jobCleanUpPolicy, err := engineDetails.DeleteJobAccordingToJobCleanUpPolicy(&experiment, clients)
if err != nil {
klog.V(0).Infof("Unable to Delete ChaosExperiment Job due to: %v", err)
}
recorder.ExperimentJobCleanUp(experiment, jobCleanUpPolicy)
recorder.ExperimentJobCleanUp(&experiment, jobCleanUpPolicy)
}
}
12 changes: 10 additions & 2 deletions pkg/utils/experimentHelpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,14 @@ import (
"k8s.io/klog"
)

func CreateExperimentList(engineDetails *EngineDetails) []ExperimentDetails {
var ExperimentDetailsList []ExperimentDetails
for i, _ := range engineDetails.Experiments {
ExperimentDetailsList = append(ExperimentDetailsList, NewExperimentDetails(engineDetails, i))
}
return ExperimentDetailsList
}

//SetValueFromChaosExperiment sets value in experimentDetails struct from chaosExperiment
func (expDetails *ExperimentDetails) SetValueFromChaosExperiment(clients ClientSets, engine *EngineDetails) error {
if err := expDetails.SetImage(clients); err != nil {
Expand Down Expand Up @@ -44,7 +52,7 @@ func (expDetails *ExperimentDetails) SetENV(engineDetails EngineDetails, clients
}

// NewExperimentDetails initilizes the structure
func NewExperimentDetails(engineDetails *EngineDetails, i int) *ExperimentDetails {
func NewExperimentDetails(engineDetails *EngineDetails, i int) ExperimentDetails {
var experimentDetails ExperimentDetails
experimentDetails.Env = make(map[string]string)
experimentDetails.ExpLabels = make(map[string]string)
Expand All @@ -58,7 +66,7 @@ func NewExperimentDetails(engineDetails *EngineDetails, i int) *ExperimentDetail
randomString := RandomString()
// Setting the JobName in Experiment Realted struct
experimentDetails.JobName = experimentDetails.Name + "-" + randomString
return &experimentDetails
return experimentDetails
}

// HandleChaosExperimentExistence will check the experiment in the app namespace
Expand Down
8 changes: 5 additions & 3 deletions pkg/utils/initialPatchEngine.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,17 @@ import (
type ExperimentStatus v1alpha1.ExperimentStatuses

// InitialPatchEngine patches the chaosEngine with the initial ExperimentStatuses
func (expStatus *ExperimentStatus) InitialPatchEngine(engineDetails EngineDetails, clients ClientSets) error {
func InitialPatchEngine(engineDetails EngineDetails, clients ClientSets, experimentList []ExperimentDetails) error {

// TODO: check for the status before patching
for range engineDetails.Experiments {
for _, v := range experimentList {
expEngine, err := engineDetails.GetChaosEngine(clients)
if err != nil {
return errors.Wrapf(err, "Unable to get ChaosEngine, due to error: %v", err)
}
expEngine.Status.Experiments = append(expEngine.Status.Experiments, v1alpha1.ExperimentStatuses(*expStatus))
var expStatus ExperimentStatus
expStatus.InitialExperimentStatus(v.JobName)
expEngine.Status.Experiments = append(expEngine.Status.Experiments, v1alpha1.ExperimentStatuses(expStatus))
_, updateErr := clients.LitmusClient.LitmuschaosV1alpha1().ChaosEngines(engineDetails.EngineNamespace).Update(expEngine)
if updateErr != nil {
return errors.Wrapf(err, "Unable to update ChaosEngine in namespace: %v, due to error: %v", engineDetails.EngineNamespace, err)
Expand Down
4 changes: 2 additions & 2 deletions pkg/utils/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import (
)

// InitialExperimentStatus fills up ExperimentStatus Structure with InitialValues
func (expStatus *ExperimentStatus) InitialExperimentStatus(experimentDetails *ExperimentDetails) {
expStatus.Name = experimentDetails.JobName
func (expStatus *ExperimentStatus) InitialExperimentStatus(jobName string) {
expStatus.Name = jobName
expStatus.Status = "Waiting for Job Creation"
expStatus.Verdict = "Waiting"
expStatus.LastUpdateTime = metav1.Now()
Expand Down

0 comments on commit 53bcf3b

Please sign in to comment.