Skip to content

Commit

Permalink
ApiServices: Use retry support for most GET, PUT, POST and DELETE api…
Browse files Browse the repository at this point in the history
… calls

Useful for retrying any api calls that fail for infra reasons.
Added base class for Ssl actions.
Added support for saving a cert to a pem file.

Testing Done: Manual testing.
Bug Number: none
Reviewed by: trivial
Run Pipeline: NO


Former-commit-id: 245b298
  • Loading branch information
damienbiggs committed May 13, 2021
1 parent 471745b commit e48927c
Show file tree
Hide file tree
Showing 24 changed files with 307 additions and 149 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public class CommitConfig {
@ConfigurableProperty(help = "Label for no bug number")
public String noBugNumberLabel;

@ConfigurableProperty(help = "Label for trivial reviewer")
@ConfigurableProperty(help = "Label for trivial reviewer", commandLine = "--trivial-reviewer-label")
public String trivialReviewerLabel;

@ConfigurableProperty(help = "Label for no reviewer")
Expand Down
12 changes: 12 additions & 0 deletions config/src/main/java/com/vmware/config/section/VcdConfig.java
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package com.vmware.config.section;

import java.util.Arrays;
import java.util.List;

import com.vmware.config.ConfigurableProperty;
import com.vmware.util.StringUtils;

public class VcdConfig {
@ConfigurableProperty(help = "Url for Vcloud Director")
Expand Down Expand Up @@ -58,4 +60,14 @@ public class VcdConfig {

@ConfigurableProperty(commandLine = "--use-owned-vapps-only", help = "Only use a Vapp owned by the user, no file based Vapps")
public boolean useOwnedVappsOnly;

@ConfigurableProperty(commandLine = "--query-filter", help = "Optional filter to use when querying Vapps or VMs")
public String queryFilter;

public String[] queryFilters() {
if (StringUtils.isEmpty(queryFilter)) {
return new String[0];
}
return new String[] {queryFilter};
}
}
63 changes: 63 additions & 0 deletions core/src/main/java/com/vmware/action/base/BaseSslAction.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package com.vmware.action.base;

import java.io.File;
import java.io.FileInputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.Provider;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;

import com.vmware.action.BaseAction;
import com.vmware.config.WorkflowConfig;
import com.vmware.util.StringUtils;
import com.vmware.util.exception.FatalException;

public abstract class BaseSslAction extends BaseAction {
public BaseSslAction(WorkflowConfig config) {
super(config);
}

protected KeyStore loadKeyStore(File keystoreFile) {
List<String> keystoreTypes = new ArrayList<>(Arrays.asList("JCEKS", "PKCS12", "JKS"));
if (sslConfig.additionalKeystoreTypes != null) {
keystoreTypes.addAll(sslConfig.additionalKeystoreTypes.keySet());
}
return keystoreTypes.stream().map(keystoreType -> {
try {
KeyStore privateKS = createKeyStore(keystoreType);
privateKS.load(new FileInputStream(keystoreFile), sslConfig.keystorePassword.toCharArray());
log.info("Loaded keystore {} of type {}", keystoreFile.getAbsolutePath(), keystoreType);
return privateKS;
} catch (Throwable t) {
log.debug("Failed with keystore type {}\n{}", keystoreType, StringUtils.exceptionAsString(t));
return null;
}
}).filter(Objects::nonNull).findFirst().orElseThrow(() -> new FatalException("Unable to load keystore file {}",
keystoreFile.getPath(), sslConfig.keystorePassword));
}

protected KeyStore createKeyStore(String keystoreType)
throws MalformedURLException, ClassNotFoundException, KeyStoreException, IllegalAccessException, InstantiationException {
if (sslConfig.additionalKeystoreTypes == null || !sslConfig.additionalKeystoreTypes.containsKey(keystoreType)) {
return KeyStore.getInstance(keystoreType);
}
List<String> providerInfo = sslConfig.additionalKeystoreTypes.get(keystoreType);

failIfTrue(providerInfo == null || providerInfo.size() != 2, "Invalid value " + providerInfo + " for keystore " + keystoreType);
String filePath = replacementVariables.replaceVariablesInValue(providerInfo.get(1));
File providerJarFile = new File(filePath);
failIfTrue(!providerJarFile.exists(), "file " + providerJarFile.getAbsolutePath() + " path not set");

log.debug("Using jar file {} for provider {}", providerJarFile.getAbsolutePath(), providerInfo.get(0));
URL jarUrl = new URL("jar:file:" + providerJarFile.getAbsolutePath() + "!/");
URLClassLoader classLoader = URLClassLoader.newInstance(new URL[] {jarUrl}, getClass().getClassLoader());
Provider provider = (Provider) classLoader.loadClass(providerInfo.get(0)).newInstance();
return KeyStore.getInstance(keystoreType, provider);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,13 @@
import com.vmware.util.exception.FatalException;
import com.vmware.util.logging.LogLevel;

import static com.vmware.util.StringUtils.convertToPem;
import static com.vmware.util.StringUtils.LINE_SEPARATOR;
import static com.vmware.util.StringUtils.convertToPem;

@ActionDescription(value = "Created a self signed cert. The host of the specified url is used as the CN value.",
configFlagsToExcludeFromCompleter = "--keystore-type")
public class CreateSelfSignedCertForUrl extends BaseAction {

public final static String LINE_SEPARATOR = System.getProperty("line.separator");

public CreateSelfSignedCertForUrl(WorkflowConfig config) {
super(config);
super.addFailWorkflowIfBlankProperties("sourceUrl");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package com.vmware.action.filesystem;

import java.io.File;
import java.io.FileInputStream;
import java.security.Key;
import java.security.KeyPair;
import java.security.KeyStore;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;

import com.vmware.action.BaseAction;
import com.vmware.action.base.BaseSslAction;
import com.vmware.config.ActionDescription;
import com.vmware.config.WorkflowConfig;
import com.vmware.util.IOUtils;
import com.vmware.util.StringUtils;
import com.vmware.util.exception.FatalException;

import static com.vmware.util.StringUtils.LINE_SEPARATOR;

@ActionDescription("Loads a certificate from a keystore")
public class LoadCertFromKeystore extends BaseSslAction {
public LoadCertFromKeystore(WorkflowConfig config) {
super(config);
super.addFailWorkflowIfBlankProperties("sourceFile", "keystoreAlias", "keystoreAliasPassword", "keystorePassword");
}

@Override
protected void failWorkflowIfConditionNotMet() {
super.failWorkflowIfConditionNotMet();
super.failIfTrue(!new File(fileSystemConfig.sourceFile).exists(), "Source file " + fileSystemConfig.sourceFile + " does not exist");
}

@Override
public void process() {
try {
log.info("Loading alias {} from keystore {}", sslConfig.keystoreAlias, fileSystemConfig.sourceFile);
KeyStore keystore = loadKeyStore(new File(fileSystemConfig.sourceFile));
Key privateKey = keystore.getKey(sslConfig.keystoreAlias, sslConfig.keystoreAliasPassword.toCharArray());
Certificate cert = keystore.getCertificate(sslConfig.keystoreAlias);

fileSystemConfig.fileData = StringUtils.convertToPem(privateKey) + LINE_SEPARATOR + StringUtils.convertToPem(cert);
} catch (Exception e) {
if (e instanceof RuntimeException) {
throw (RuntimeException) e;
} else {
throw new RuntimeException(e);
}
}
}
}
53 changes: 4 additions & 49 deletions core/src/main/java/com/vmware/action/filesystem/SaveCert.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,8 @@

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.StandardCharsets;
import java.security.AlgorithmParameters;
import java.security.InvalidKeyException;
Expand All @@ -18,7 +14,6 @@
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.security.Provider;
import java.security.PublicKey;
import java.security.SecureRandom;
import java.security.SignatureException;
Expand All @@ -30,12 +25,8 @@
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.RSAPublicKeySpec;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Calendar;
import java.util.List;
import java.util.Objects;
import java.util.Random;

import javax.crypto.Cipher;
Expand All @@ -45,12 +36,11 @@
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.PBEParameterSpec;

import com.vmware.action.BaseAction;
import com.vmware.action.base.BaseSslAction;
import com.vmware.config.ActionDescription;
import com.vmware.config.WorkflowConfig;
import com.vmware.util.IOUtils;
import com.vmware.util.StringUtils;
import com.vmware.util.exception.FatalException;

import static com.vmware.util.StringUtils.BEGIN_PRIVATE_KEY;
import static com.vmware.util.StringUtils.END_PRIVATE_KEY;
Expand All @@ -59,10 +49,10 @@

@ActionDescription(value = "Saves the loaded cert. If keystore password is set, cert is saved to a keystore, otherwise as pem files.",
configFlagsToAlwaysExcludeFromCompleter = {"--cipher-salt-length", "--new-keystore-type"})
public class SaveCert extends BaseAction {
public class SaveCert extends BaseSslAction {
public SaveCert(WorkflowConfig config) {
super(config);
super.addSkipActionIfBlankProperties("fileData", "destinationFile", "keystoreAlias", "keystorePassword");
super.addSkipActionIfBlankProperties("fileData", "destinationFile", "keystoreAliasPassword");
}

@Override
Expand Down Expand Up @@ -112,6 +102,7 @@ private void addCertToKeyStoreFile()
throws ClassNotFoundException, KeyStoreException, IllegalAccessException, InstantiationException, CertificateException, NoSuchAlgorithmException,
InvalidKeyException, NoSuchProviderException, SignatureException, InvalidKeySpecException, IOException {
failIfEmpty("keystoreAliasPassword", "keystoreAliasPassword not set");
failIfEmpty("keystoreAlias", "keystoreAlias not set");
File keystoreFile = new File(fileSystemConfig.destinationFile);
KeyStore privateKS;
if (keystoreFile.exists()) {
Expand All @@ -134,43 +125,7 @@ private void addCertToKeyStoreFile()
}


private KeyStore loadKeyStore(File keystoreFile) {
List<String> keystoreTypes = new ArrayList<>(Arrays.asList("JCEKS", "PKCS12", "JKS"));
if (sslConfig.additionalKeystoreTypes != null) {
keystoreTypes.addAll(sslConfig.additionalKeystoreTypes.keySet());
}
return keystoreTypes.stream().map(keystoreType -> {
try {
KeyStore privateKS = createKeyStore(keystoreType);
privateKS.load(new FileInputStream(keystoreFile), sslConfig.keystorePassword.toCharArray());
log.info("Loaded keystore {} of type {}", keystoreFile.getAbsolutePath(), keystoreType);
return privateKS;
} catch (Throwable t) {
log.debug("Failed with keystore type {}\n{}", keystoreType, StringUtils.exceptionAsString(t));
return null;
}
}).filter(Objects::nonNull).findFirst().orElseThrow(() -> new FatalException("Unable to load keystore file {}",
keystoreFile.getPath(), sslConfig.keystorePassword));
}

private KeyStore createKeyStore(String keystoreType)
throws MalformedURLException, ClassNotFoundException, KeyStoreException, IllegalAccessException, InstantiationException {
if (sslConfig.additionalKeystoreTypes == null || !sslConfig.additionalKeystoreTypes.containsKey(keystoreType)) {
return KeyStore.getInstance(keystoreType);
}
List<String> providerInfo = sslConfig.additionalKeystoreTypes.get(keystoreType);

failIfTrue(providerInfo == null || providerInfo.size() != 2, "Invalid value " + providerInfo + " for keystore " + keystoreType);
String filePath = replacementVariables.replaceVariablesInValue(providerInfo.get(1));
File providerJarFile = new File(filePath);
failIfTrue(!providerJarFile.exists(), "file " + providerJarFile.getAbsolutePath() + " path not set");

log.debug("Using jar file {} for provider {}", providerJarFile.getAbsolutePath(), providerInfo.get(0));
URL jarUrl = new URL("jar:file:" + providerJarFile.getAbsolutePath() + "!/");
URLClassLoader classLoader = URLClassLoader.newInstance(new URL[] {jarUrl}, getClass().getClassLoader());
Provider provider = (Provider) classLoader.loadClass(providerInfo.get(0)).newInstance();
return KeyStore.getInstance(keystoreType, provider);
}

private X509Certificate createCertificate() throws CertificateException, NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException, SignatureException {
String certOutput = StringUtils.findStringWithStartAndEnd(fileSystemConfig.fileData, BEGIN_CERT, END_CERT);
Expand Down
45 changes: 45 additions & 0 deletions core/src/main/java/com/vmware/action/vcd/DeleteVMs.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package com.vmware.action.vcd;

import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import com.vmware.action.BaseAction;
import com.vmware.config.ActionDescription;
import com.vmware.config.WorkflowConfig;
import com.vmware.util.CollectionUtils;
import com.vmware.util.input.InputUtils;
import com.vmware.vcd.Vcd;
import com.vmware.vcd.domain.QueryResultVMType;
import com.vmware.vcd.domain.QueryResultVMsType;
import com.vmware.vcd.domain.TaskType;

@ActionDescription("Action for bulk deleting of VMs")
public class DeleteVMs extends BaseAction {
public DeleteVMs(WorkflowConfig config) {
super(config);
}

@Override
public void process() {
Vcd vcd = serviceLocator.getVcd();
QueryResultVMsType vmRecords = vcd.queryVms(vcdConfig.queryFilters());
if (CollectionUtils.isEmpty(vmRecords.record)) {
log.info("No VMs found");
return;
}
List<String> choices = vmRecords.record.stream().map(QueryResultVMType::getLabel).collect(Collectors.toList());
List<Integer> vmIndexes = InputUtils.readSelections(choices, "Select VMs to delete", false);

String vmsToDelete = vmIndexes.stream().map(choices::get).collect(Collectors.joining(","));
log.info("VMs {} will be deleted", vmsToDelete);
String confirmation = InputUtils.readValueUntilNotBlank("Delete (Y/N)");
if ("Y".equalsIgnoreCase(confirmation)) {
vmIndexes.stream().map(i -> vmRecords.record.get(i)).forEach(vm -> {
log.info("Deleting VM {}", vm.getLabel());
TaskType deleteTask = serviceLocator.getVcd().deleteResource(vm.getSelfLink(), true);
vcd.waitForTaskToComplete(deleteTask.href, 1, TimeUnit.MINUTES);
});
}
}
}
2 changes: 1 addition & 1 deletion core/src/main/java/com/vmware/action/vcd/LoadVapps.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void checkIfActionShouldBeSkipped() {

@Override
public void process() {
QueryResultVappsType vappRecords = serviceLocator.getVcd().queryVapps();
QueryResultVappsType vappRecords = serviceLocator.getVcd().queryVapps(vcdConfig.queryFilters());
vappRecords.record.forEach(this::populatedPoweredOnVmCount);
List<QueryResultVappType> vapps = new ArrayList<>();
vapps.addAll(parseVappJsonFiles());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public class GenerateActionConfigMappings {
methodNameToConfigValueMappings.put("trackingBranchPath", singletonList("trackingBranch"));
methodNameToConfigValueMappings.put("commandLineSite", Arrays.asList("sshHost", "sshPort", "sshUsername", "sshPassword"));
methodNameToConfigValueMappings.put("determineBuildwebBranch", Arrays.asList("buildwebBranch", "useGitTrackingBranch"));
methodNameToConfigValueMappings.put("queryFilters", singletonList("queryFilter"));
}

private static final Logger log = LoggerFactory.getLogger(GenerateActionConfigMappings.class);
Expand Down
Loading

0 comments on commit e48927c

Please sign in to comment.