SkipAction: Add support for skipping workflow action based on variable

Can skip on value for variable being present or not present.
Add support for identifying jenkins build number from Vapp / VM name based on pattern.
Create new release version for May 21st

appLoader/pom.xml

@@ -22,8 +22,8 @@
                         <manifestEntries>
                             <appMainClass>com.vmware.WorkflowRunner</appMainClass>
                             <deleteOldReleaseJarPattern>workflowTools-[\w_]+.jar</deleteOldReleaseJarPattern>
-                            <releaseJarName>workflowTools-May_14_2021.jar</releaseJarName>
-                            <releaseUrl>https://github.com/vmware/workflowTools/releases/download/May_14_2021/workflowTools.jar</releaseUrl>
+                            <releaseJarName>workflowTools-May_21_2021.jar</releaseJarName>
+                            <releaseUrl>https://github.com/vmware/workflowTools/releases/download/May_21_2021/workflowTools.jar</releaseUrl>
                         </manifestEntries>
                     </archive>
                 </configuration>

appLoader/src/main/java/com/vmware/WorkflowAppLoader.java

@@ -36,6 +36,7 @@ public class WorkflowAppLoader {
     public static void main(String[] args) {
         WorkflowAppLoader loader = new WorkflowAppLoader(args);
         loader.downloadJarFileIfNeeded();
+
         loader.executeWorkflowJar();
     }
 
@@ -47,8 +48,7 @@ public WorkflowAppLoader(String[] args) {
         this.releaseDirectory = manifestAttributes.containsKey("releaseDirectory")
                 ? manifestAttributes.get("releaseDirectory") : System.getProperty("java.io.tmpdir");
         this.releaseJar = new File(this.releaseDirectory + File.separator + manifestAttributes.get("releaseJarName"));
-        Optional<String> testReleaseJarPath = getArgValue("--test-release-jar");
-        this.testReleaseJar = testReleaseJarPath.map(File::new).orElse(null);
+        this.testReleaseJar = getArgValue("--test-release-jar").map(File::new).orElse(null);
     }
 
     public void executeWorkflowJar() {

config/src/main/java/com/vmware/config/section/FileSystemConfig.java

@@ -70,6 +70,15 @@ public class FileSystemConfig {
     @ConfigurableProperty(commandLine = "--output-variable-name", help = "Name of variable used for output")
     public String outputVariableName;
 
+    @ConfigurableProperty(commandLine = "--variable", help = "Name of variable to use")
+    public String variable;
+
+    @ConfigurableProperty(commandLine = "--skip-if-variable-set", help = "Skips action if the specified variable has a value")
+    public boolean skipIfVariableSet;
+
+    @ConfigurableProperty(commandLine = "--skip-if-variable-not-set", help = "Skips action if the specified variable does not have a value")
+    public boolean skipIfVariableNotSet;
+
     public boolean databaseConfigured() {
         return Stream.of(databaseUrl, databaseUsername, databasePassword).allMatch(StringUtils::isNotBlank);
     }

config/src/main/java/com/vmware/config/section/JenkinsConfig.java

@@ -86,8 +86,11 @@ public class JenkinsConfig {
     public String commitComparisonUrl;
 
     public boolean hasConfiguredArtifact() {
-        return StringUtils.isNotEmpty(jobArtifact) && jobBuildNumber != null
-                && (StringUtils.isNotEmpty(jobWithArtifact) ||  (jobsDisplayNames != null && jobsDisplayNames.length == 1));
+        return hasConfiguredArtifactWithoutBuildNumber() && jobBuildNumber != null;
+    }
+
+    public boolean hasConfiguredArtifactWithoutBuildNumber() {
+        return StringUtils.isNotEmpty(jobArtifact) && (StringUtils.isNotEmpty(jobWithArtifact) ||  (jobsDisplayNames != null && jobsDisplayNames.length == 1));
     }
 
     public void addJenkinsParametersFromConfigValues(Map<String, String> configValues, boolean overwriteJenkinsParameters) {

config/src/main/java/com/vmware/config/section/VcdConfig.java

@@ -64,6 +64,9 @@ public class VcdConfig {
     @ConfigurableProperty(commandLine = "--query-filter", help = "Optional filter to use when querying Vapps or VMs")
     public String queryFilter;
 
+    @ConfigurableProperty(help = "Pattern for parsing build number for Vapp / Vm name")
+    public String buildNumberInNamePattern;
+
     public String[] queryFilters() {
         if (StringUtils.isEmpty(queryFilter)) {
             return new String[0];

core/src/main/java/com/vmware/Workflow.java

@@ -346,7 +346,7 @@ private void dryRunActions(List<WorkflowAction> actions) {
                 sectionPadder.infoTitle();
                 sectionPadder = null;
             }
-            configOptions.addAll(configMappings.getUsableConfigValuesForAction(action));
+            configOptions.addAll(configMappings.getAutoCompleteValuesForAction(action));
             configValuesToRemove.addAll(action.configFlagsToAlwaysRemoveFromCompleter());
             config.addGeneratedVariables();
             List<WorkflowParameter> params = action.getOverriddenConfigValues();

core/src/main/java/com/vmware/action/BaseAction.java

@@ -5,6 +5,7 @@
 import java.util.Set;
 
 import com.vmware.ServiceLocator;
+import com.vmware.config.ActionDescription;
 import com.vmware.config.ReplacementVariables;
 import com.vmware.config.WorkflowConfig;
 import com.vmware.config.WorkflowField;
@@ -37,6 +38,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+@ActionDescription(value = "", configFlagsToExcludeFromCompleter = {"--variable", "--skip-if-variable-set", "--skip-if-variable-not-set"})
 public abstract class BaseAction implements Action {
 
     protected Logger log = LoggerFactory.getLogger(this.getClass());
@@ -123,6 +125,16 @@ public void asyncSetup() {
     @Override
     public void checkIfActionShouldBeSkipped() {
         skipActionIfBlankProperties.forEach(this::skipActionIfUnset);
+        if (fileSystemConfig.skipIfVariableSet) {
+            failIfUnset("variable");
+            skipActionIfTrue(replacementVariables.hasVariable(fileSystemConfig.variable),
+                    fileSystemConfig.variable + " variable is set");
+        }
+        if (fileSystemConfig.skipIfVariableNotSet) {
+            failIfUnset("variable");
+            skipActionIfTrue(!replacementVariables.hasVariable(fileSystemConfig.variable),
+                    fileSystemConfig.variable + " variable is not set");
+        }
     }
 
     @Override

core/src/main/java/com/vmware/action/base/BaseCommitAction.java

@@ -133,7 +133,7 @@ protected String determineSandboxBuildNumber(String buildDisplayName) {
     }
 
     protected String jobWithArtifactName() {
-        if (!jenkinsConfig.hasConfiguredArtifact()) {
+        if (!jenkinsConfig.hasConfiguredArtifactWithoutBuildNumber()) {
             return null;
         }
         return StringUtils.isNotEmpty(jenkinsConfig.jobWithArtifact) ? jenkinsConfig.jobWithArtifact : jenkinsConfig.jobsDisplayNames[0];

core/src/main/java/com/vmware/action/filesystem/DecryptProperty.java

@@ -16,7 +16,7 @@
 
 import static com.vmware.util.ArrayUtils.subSection;
 
-@ActionDescription("Decrypts the specified property to the specified variable name.")
+@ActionDescription(value = "Decrypts the specified property to the specified variable name.")
 public class DecryptProperty extends BaseAction {
 
     public DecryptProperty(WorkflowConfig config) {

core/src/main/java/com/vmware/action/filesystem/ExecuteLocalSqlStatementUsingPsql.java

@@ -4,6 +4,9 @@
 import com.vmware.config.ActionDescription;
 import com.vmware.config.WorkflowConfig;
 import com.vmware.util.CommandLineUtils;
+import com.vmware.util.MatcherUtils;
+import com.vmware.util.StringUtils;
+import com.vmware.util.exception.FatalException;
 import com.vmware.util.logging.LogLevel;
 
 @ActionDescription("Execute a sql statement locally using psql.")
@@ -17,9 +20,24 @@ public ExecuteLocalSqlStatementUsingPsql(WorkflowConfig config) {
     @Override
     public void process() {
         log.info("Executing sql statement {} using schema {}", fileSystemConfig.sqlStatement, fileSystemConfig.databaseSchemaName);
-        String command = String.format("psql -U %s -d %s -c \"%s\"", fileSystemConfig.databaseUsername,
+        String command = String.format("psql -t -U %s -d %s -c \"%s\"", fileSystemConfig.databaseUsername,
                 fileSystemConfig.databaseSchemaName, fileSystemConfig.sqlStatement);
-        CommandLineUtils.executeCommand(command, LogLevel.INFO);
+        String output = StringUtils.trim(CommandLineUtils.executeCommand(command, LogLevel.INFO));
 
+        if (StringUtils.textStartsWithValue(output, "psql: error")) {
+            throw new FatalException(output);
+        }
+
+        String missingTable = MatcherUtils.singleMatch(output, "relation \"([\\w_]+)\" does not exist");
+        if (StringUtils.isNotBlank(missingTable) && StringUtils.isNotBlank(fileSystemConfig.outputVariableName)) {
+            log.info("Skipping setting value for output variable {} as table {} does not exist", fileSystemConfig.outputVariableName, missingTable);
+        } else if (StringUtils.isNotBlank(missingTable)) {
+            throw new FatalException(output);
+        } else if (StringUtils.isNotBlank(output) && StringUtils.isNotBlank(fileSystemConfig.outputVariableName)) {
+            log.info("Saving output {} to output variable {}", output, fileSystemConfig.outputVariableName);
+            replacementVariables.addVariable(fileSystemConfig.outputVariableName, output, false);
+        } else if (StringUtils.isNotBlank(fileSystemConfig.outputVariableName)) {
+            log.info("No value to set to output variable {}", fileSystemConfig.outputVariableName);
+        }
     }
 }

core/src/main/java/com/vmware/action/jenkins/DownloadBuildArtifact.java

@@ -2,6 +2,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 import com.vmware.action.base.BaseCommitWithJenkinsBuildsAction;
 import com.vmware.config.ActionDescription;
@@ -11,7 +12,14 @@
 import com.vmware.jenkins.domain.JobBuild;
 import com.vmware.util.FileUtils;
 import com.vmware.util.IOUtils;
+import com.vmware.util.MatcherUtils;
+import com.vmware.util.StringUtils;
+import com.vmware.util.UrlUtils;
+import com.vmware.util.exception.CancelException;
 import com.vmware.util.input.InputUtils;
+import com.vmware.util.logging.LogLevel;
+import com.vmware.vcd.domain.QueryResultVMsType;
+import com.vmware.vcd.domain.QueryResultVappType;
 
 @ActionDescription("Download a specified artifact from a jenkins build.")
 public class DownloadBuildArtifact extends BaseCommitWithJenkinsBuildsAction {
@@ -20,14 +28,6 @@ public DownloadBuildArtifact(WorkflowConfig config) {
         super.addFailWorkflowIfBlankProperties("jobArtifact");
     }
 
-    @Override
-    protected void failWorkflowIfConditionNotMet() {
-        super.failWorkflowIfConditionNotMet();
-        if (!jenkinsConfig.hasConfiguredArtifact() && draft.jobBuildsMatchingUrl(jenkinsConfig.jenkinsUrl).isEmpty()) {
-            exitDueToFailureCheck("Jenkins artifact is not configured and there are no builds in the commit testing done section");
-        }
-    }
-
     @Override
     public void process() {
         JobBuild buildDetails = getJobBuildDetails();
@@ -55,11 +55,26 @@ private JobBuild getJobBuildDetails() {
     }
 
     private JobBuild determineBuildToUse() {
+        if (vappData.getSelectedVapp() != null && StringUtils.isNotBlank(vcdConfig.buildNumberInNamePattern)
+                && jenkinsConfig.hasConfiguredArtifactWithoutBuildNumber()) {
+            QueryResultVappType selectedVapp = vappData.getSelectedVapp();
+            JobBuild matchingJobBuild = getJobBuildForName(selectedVapp.name);
+            if (matchingJobBuild == null) {
+                QueryResultVMsType vms = serviceLocator.getVcd().queryVmsForVapp(selectedVapp.parseIdFromRef());
+                matchingJobBuild = vms.record.stream().map(vm -> getJobBuildForName(vm.name)).filter(Objects::nonNull).findFirst().orElse(null);
+            }
+            if (matchingJobBuild != null) {
+                return matchingJobBuild;
+            }
+        }
+
         List<JobBuild> matchingBuilds = draft.jobBuildsMatchingUrl(jenkinsConfig.jenkinsUrl);
         if (draft.selectedBuild != null) {
             return matchingBuilds.get(draft.selectedBuild);
         }
-        if (matchingBuilds.size() == 1) {
+        if (matchingBuilds.isEmpty()) {
+            throw new CancelException(LogLevel.ERROR, "No matching builds found for url " + jenkinsConfig.jenkinsUrl);
+        } else if (matchingBuilds.size() == 1) {
             log.info("Using build {} as it is the only Jenkins build", matchingBuilds.get(0).name);
             draft.selectedBuild = 0;
             return matchingBuilds.get(0);
@@ -71,4 +86,16 @@ private JobBuild determineBuildToUse() {
             return matchingBuilds.get(selection);
         }
     }
+
+    private JobBuild getJobBuildForName(String name) {
+        String buildNumber = MatcherUtils.singleMatch(name, vcdConfig.buildNumberInNamePattern);
+        if (StringUtils.isInteger(buildNumber)) {
+            String jobName = jobWithArtifactName();
+            log.info("Found matching build {} from name {} for job {}", buildNumber, name, jobName);
+            String jobUrl = UrlUtils.addRelativePaths(jenkins.baseUrl, "job", jobName);
+            return new JobBuild(Integer.parseInt(buildNumber), jobUrl);
+        } else {
+            return null;
+        }
+    }
 }

core/src/main/java/com/vmware/action/jenkins/DownloadBuildArtifactIfNeeded.java

@@ -10,18 +10,12 @@ public DownloadBuildArtifactIfNeeded(WorkflowConfig config) {
         super(config);
     }
 
-    @Override
-    protected void failWorkflowIfConditionNotMet() {
-    }
-
     @Override
     public void checkIfActionShouldBeSkipped() {
         super.checkIfActionShouldBeSkipped();
-        if (StringUtils.isNotBlank(jenkinsConfig.jobArtifact) && StringUtils.isNotBlank(fileSystemConfig.fileData) && !jenkinsConfig.alwaysDownload) {
+        if (StringUtils.isNotBlank(fileSystemConfig.fileData) && !jenkinsConfig.alwaysDownload) {
             skipActionDueTo("artifact {} has already been loaded, use flag --always-download to override and select a build to use",
                     jenkinsConfig.jobArtifact);
-        } else if (!jenkinsConfig.hasConfiguredArtifact() && draft.jobBuildsMatchingUrl(jenkinsConfig.jenkinsUrl).isEmpty()) {
-            skipActionDueTo("Jenkins artifact is not configured and there are no builds in the commit testing done section");
         }
     }
 

core/src/main/java/com/vmware/action/vcd/LoadVapps.java

@@ -52,9 +52,6 @@ private void populatedPoweredOnVmCount(QueryResultVappType queryResultVappType)
             queryResultVappType.poweredOnVmCount = queryResultVappType.otherAttributes.numberOfVMs;
         } else if ("MIXED".equalsIgnoreCase(queryResultVappType.status) && queryResultVappType.isOwnedByWorkflowUser()) {
             String vappId = queryResultVappType.parseIdFromRef();
-            if (vappId.startsWith("vapp-")) {
-                vappId = vappId.substring("vapp-".length());
-            }
             QueryResultVMsType vmsForVapp = serviceLocator.getVcd().queryVmsForVapp(vappId);
             queryResultVappType.poweredOnVmCount = (int) vmsForVapp.record.stream().filter(QueryResultVMType::isPoweredOn).count();
         }

core/src/main/java/com/vmware/config/WorkflowAction.java

@@ -22,6 +22,7 @@
 import com.vmware.util.exception.SkipActionException;
 
 import static com.vmware.util.StringUtils.pluralizeDescription;
+import static java.util.Arrays.asList;
 
 public class WorkflowAction implements Action {
     private String sectionName;
@@ -49,28 +50,21 @@ public List<String> configFlagsToRemoveFromCompleter() {
         List<String> configFlagsToRemoveFromCompleter = new ArrayList<>();
         do {
             ActionDescription actionDescription = classToCheck.getAnnotation(ActionDescription.class);
-            configFlagsToRemoveFromCompleter.addAll(Arrays.asList(actionDescription.configFlagsToExcludeFromCompleter()));
-            if (classToCheck.getSuperclass().getAnnotation(ActionDescription.class) != null) {
-                classToCheck = (Class<? extends BaseAction>) classToCheck.getSuperclass();
-            } else {
-                classToCheck = null;
-            }
-        } while (classToCheck != null);
+            configFlagsToRemoveFromCompleter.addAll(asList(actionDescription.configFlagsToExcludeFromCompleter()));
+            classToCheck = (Class<? extends BaseAction>) classToCheck.getSuperclass();
+        } while (classToCheck != BaseAction.class);
         return configFlagsToRemoveFromCompleter;
     }
 
     public List<String> configFlagsToAlwaysRemoveFromCompleter() {
         Class<? extends BaseAction> classToCheck = actionClass;
+
         List<String> configFlagsToAlwaysRemoveFromCompleter = new ArrayList<>();
         do {
             ActionDescription actionDescription = classToCheck.getAnnotation(ActionDescription.class);
-            configFlagsToAlwaysRemoveFromCompleter.addAll(Arrays.asList(actionDescription.configFlagsToAlwaysExcludeFromCompleter()));
-            if (classToCheck.getSuperclass().getAnnotation(ActionDescription.class) != null) {
-                classToCheck = (Class<? extends BaseAction>) classToCheck.getSuperclass();
-            } else {
-                classToCheck = null;
-            }
-        } while (classToCheck != null);
+            configFlagsToAlwaysRemoveFromCompleter.addAll(asList(actionDescription.configFlagsToAlwaysExcludeFromCompleter()));
+            classToCheck = (Class<? extends BaseAction>) classToCheck.getSuperclass();
+        } while (classToCheck != BaseAction.class);
         return configFlagsToAlwaysRemoveFromCompleter;
     }
 
@@ -159,21 +153,24 @@ public Set<String> getWorkflowParameterNames() {
         return overriddenConfigValues.stream().map(WorkflowParameter::getName).collect(Collectors.toSet());
     }
 
-    public Set<String> getConfigValues(Map<String, List<String>> mappings) {
+    public Set<String> getConfigValues(Map<String, List<String>> mappings, boolean autoCompleteValuesOnly) {
         Set<String> configValues = new HashSet<String>();
-        Class classToGetValuesFor = actionClass;
-        while (classToGetValuesFor != Object.class) {
+        Class<? extends BaseAction> classToGetValuesFor = actionClass;
+        do {
             List<String> configValuesForClass = mappings.get(classToGetValuesFor.getSimpleName());
             if (configValuesForClass != null) {
                 configValues.addAll(configValuesForClass);
             }
             boolean ignoreSuperClass = false;
-            if (classToGetValuesFor != BaseAction.class && classToGetValuesFor.isAnnotationPresent(ActionDescription.class)) {
-                Class<? extends BaseAction> actionClass = classToGetValuesFor;
-                ignoreSuperClass = actionClass.getAnnotation(ActionDescription.class).ignoreConfigValuesInSuperclass();
+            if (classToGetValuesFor.isAnnotationPresent(ActionDescription.class)) {
+                if (autoCompleteValuesOnly) {
+                    List<String> configFlagsToExclude = asList(classToGetValuesFor.getAnnotation(ActionDescription.class).configFlagsToExcludeFromCompleter());
+                    configValues.removeIf(configFlagsToExclude::contains);
+                }
+                ignoreSuperClass = classToGetValuesFor.getAnnotation(ActionDescription.class).ignoreConfigValuesInSuperclass();
             }
-            classToGetValuesFor = ignoreSuperClass ? Object.class : classToGetValuesFor.getSuperclass();
-        }
+            classToGetValuesFor = ignoreSuperClass ? null : (Class<? extends BaseAction>) classToGetValuesFor.getSuperclass();
+        } while (classToGetValuesFor != null && classToGetValuesFor != BaseAction.class);
         return configValues;
     }
 
@@ -185,7 +182,7 @@ private void resetConfigValues() {
     }
 
     private void setWorkflowParametersForAction(ConfigMappings mappings, List<WorkflowParameter> parameters) {
-        Set<String> allowedConfigValues = mappings.getConfigValuesForAction(this);
+        Set<String> allowedConfigValues = mappings.getConfigValuesForAction(this, false);
         Set<String> unknownParameters = parameters.stream().filter(param -> !paramIsAllowed(allowedConfigValues, param.getName()))
                 .map(WorkflowParameter::getName).collect(Collectors.toSet());