Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support EKSA bundles #895

Merged
merged 3 commits into from
May 16, 2024
+155 −26
Merged

Support EKSA bundles #895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d41033d
cluster: add function to download eks-a bundle file
gthao313 May 6, 2024
5d504c2
vmware: add bundles-override argument to eks-a
gthao313 May 6, 2024
befef74
metal: add bundles-override argument to eks-a
gthao313 May 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion bottlerocket/agents/src/bin/metal-k8s-cluster-resource-agent/metal_k8s_cluster_provider.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ use agent_utils::aws::aws_config;
use agent_utils::base64_decode_write_file;
use agent_utils::ssm::{create_ssm_activation, ensure_ssm_service_role, wait_for_ssm_ready};
use bottlerocket_agents::clusters::{
install_eks_a_binary, retrieve_workload_cluster_kubeconfig, write_validate_mgmt_kubeconfig,
download_eks_a_bundle, install_eks_a_binary, retrieve_workload_cluster_kubeconfig,
write_validate_mgmt_kubeconfig,
};
use bottlerocket_types::agent_config::{
CustomUserData, MetalK8sClusterConfig, AWS_CREDENTIALS_SECRET_NAME,
Expand Down Expand Up @@ -168,6 +169,7 @@ impl Create for MetalK8sClusterCreator {
)?;

let mgmt_kubeconfig_path = format!("{}/mgmt.kubeconfig", WORKING_DIR);
let bundle_manifest_path = format!("{}/bundles.yaml", WORKING_DIR);
let eksa_config_path = format!("{}/cluster.yaml", WORKING_DIR);
let hardware_csv_path = format!("{}/hardware.csv", WORKING_DIR);
base64_decode_write_file(&spec.configuration.hardware_csv_base64, &hardware_csv_path)
Expand Down Expand Up @@ -234,6 +236,12 @@ impl Create for MetalK8sClusterCreator {
)?;

install_eks_a_binary(&spec.configuration.eks_a_release_manifest_url, &resources).await?;
download_eks_a_bundle(
&spec.configuration.eks_a_release_manifest_url,
&bundle_manifest_path,
&resources,
)
.await?;

info!("Creating cluster");
memo.current_status = "Creating cluster".to_string();
Expand All @@ -252,6 +260,7 @@ impl Create for MetalK8sClusterCreator {
let status = Command::new("eksctl")
.args(["anywhere", "create", "cluster"])
.args(["--kubeconfig", &mgmt_kubeconfig_path])
.args(["--bundles-override", &bundle_manifest_path])
.args(["-f", &eksa_config_path])
.args(["--hardware-csv", &hardware_csv_path])
.arg("--skip-ip-check")
Expand Down Expand Up @@ -529,6 +538,7 @@ impl Destroy for MetalK8sClusterDestroyer {

// Set the cluster deletion configs paths
let mgmt_kubeconfig_path = format!("{}/mgmt.kubeconfig", WORKING_DIR);
let bundle_manifest_path = format!("{}/bundle.yaml", WORKING_DIR);
let eksa_config_path = format!("{}/cluster.yaml", WORKING_DIR);
let workload_kubeconfig_path =
format!("{}/{}-eks-a-cluster.kubeconfig", WORKING_DIR, cluster_name);
Expand All @@ -539,6 +549,12 @@ impl Destroy for MetalK8sClusterDestroyer {
.configuration;

install_eks_a_binary(&configuration.eks_a_release_manifest_url, &resources).await?;
download_eks_a_bundle(
&configuration.eks_a_release_manifest_url,
&bundle_manifest_path,
&resources,
)
.await?;

base64_decode_write_file(
&configuration.mgmt_cluster_kubeconfig_base64,
Expand Down Expand Up @@ -578,6 +594,7 @@ impl Destroy for MetalK8sClusterDestroyer {
let status = Command::new("eksctl")
.args(["anywhere", "delete", "cluster"])
.args(["--kubeconfig", &mgmt_kubeconfig_path])
.args(["--bundles-override", &bundle_manifest_path])
.args(["-f", &eksa_config_path])
.args(["--w-config", &workload_kubeconfig_path])
.args(["-v", "4"])
Expand Down
21 changes: 20 additions & 1 deletion ...erocket/agents/src/bin/vsphere-k8s-cluster-resource-agent/vsphere_k8s_cluster_provider.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use agent_utils::base64_decode_write_file;
use bottlerocket_agents::clusters::{
install_eks_a_binary, retrieve_workload_cluster_kubeconfig, write_validate_mgmt_kubeconfig,
download_eks_a_bundle, install_eks_a_binary, retrieve_workload_cluster_kubeconfig,
write_validate_mgmt_kubeconfig,
};
use bottlerocket_agents::constants::TEST_CLUSTER_KUBECONFIG_PATH;
use bottlerocket_agents::is_cluster_creation_required;
Expand Down Expand Up @@ -147,9 +148,16 @@ impl Create for VSphereK8sClusterCreator {
.context(resources, "Error sending cluster creation message")?;

let mgmt_kubeconfig_path = format!("{}/mgmt.kubeconfig", WORKING_DIR);
let bundle_manifest_path = format!("{}/bundles.yaml", WORKING_DIR);
let encoded_kubeconfig = if do_create {
install_eks_a_binary(&spec.configuration.eks_a_release_manifest_url, &resources)
.await?;
download_eks_a_bundle(
&spec.configuration.eks_a_release_manifest_url,
&bundle_manifest_path,
&resources,
)
.await?;

info!("Creating cluster");
memo.current_status = "Creating cluster".to_string();
Expand All @@ -166,6 +174,7 @@ impl Create for VSphereK8sClusterCreator {
create_vsphere_k8s_cluster(
&spec.configuration,
&mgmt_kubeconfig_path,
&bundle_manifest_path,
&mut resources,
&mut memo,
)
Expand Down Expand Up @@ -237,6 +246,7 @@ async fn does_cluster_exist(config: &VSphereK8sClusterConfig) -> ProviderResult<
async fn create_vsphere_k8s_cluster(
config: &VSphereK8sClusterConfig,
mgmt_kubeconfig_path: &str,
bundle_manifest_path: &str,
resources: &mut Resources,
memo: &mut ProductionMemo,
) -> ProviderResult<()> {
Expand Down Expand Up @@ -434,6 +444,7 @@ async fn create_vsphere_k8s_cluster(
let status = Command::new("eksctl")
.args(["anywhere", "create", "cluster"])
.args(["--kubeconfig", mgmt_kubeconfig_path])
.args(["--bundles-override", bundle_manifest_path])
.args(["-f", &clusterspec_path])
.args(["-v", "4"])
.stdout(Stdio::inherit())
Expand Down Expand Up @@ -672,6 +683,7 @@ impl Destroy for VSphereK8sClusterDestroyer {
memo.vm_template = "".to_string();

let mgmt_kubeconfig_path = format!("{}/mgmt.kubeconfig", WORKING_DIR);
let bundle_manifest_path = format!("{}/bundle.yaml", WORKING_DIR);
debug!("Decoding and writing out kubeconfig for the CAPI management cluster");
base64_decode_write_file(
&spec.configuration.mgmt_cluster_kubeconfig_base64,
Expand All @@ -684,6 +696,12 @@ impl Destroy for VSphereK8sClusterDestroyer {
)?;

install_eks_a_binary(&spec.configuration.eks_a_release_manifest_url, &resources).await?;
download_eks_a_bundle(
&spec.configuration.eks_a_release_manifest_url,
&bundle_manifest_path,
&resources,
)
.await?;

// For cluster deletion, EKS-A needs the workload cluster's kubeconfig at
// './${CLUSTER_NAME}/${CLUSTER_NAME}-eks-a-cluster.kubeconfig'
Expand Down Expand Up @@ -728,6 +746,7 @@ impl Destroy for VSphereK8sClusterDestroyer {
let status = Command::new("eksctl")
.args(["anywhere", "delete", "cluster"])
.args(["--kubeconfig", &mgmt_kubeconfig_path])
.args(["--bundles-override", &bundle_manifest_path])
.arg(spec.configuration.name)
.args(["-v", "4"])
.stdout(Stdio::inherit())
Expand Down
141 changes: 117 additions & 24 deletions bottlerocket/agents/src/clusters.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use serde::Deserialize;
use std::convert::TryFrom;
use std::env;
use std::fmt::Display;
use std::fs::File;
use std::path::PathBuf;

/// Write out and check CAPI management cluster is accessible and valid
Expand Down Expand Up @@ -74,30 +75,7 @@ pub async fn get_eks_a_archive_url<S>(
where
S: AsRef<str> + IntoUrl + Display,
{
let manifest = reqwest::get(eks_a_release_manifest_url.to_string())
.await
.context(
resources,
format!(
"Unable to request EKS-A release manifest '{}'",
eks_a_release_manifest_url
),
)?
.text()
.await
.context(
resources,
format!(
"Unable to retrieve EKS-A release manifest at '{}'",
eks_a_release_manifest_url
),
)?;
let deserialized_manifest = serde_yaml::Deserializer::from_str(&manifest)
.map(|config| {
serde_yaml::Value::deserialize(config)
.context(resources, "Unable to deserialize eksa config file")
})
.collect::<ProviderResult<Vec<_>>>()?;
let deserialized_manifest = get_eks_a_manifest(eks_a_release_manifest_url, resources).await?;

let latest_release_version = deserialized_manifest
.iter()
Expand Down Expand Up @@ -145,6 +123,53 @@ where
.map(|s| s.to_string())
}

pub async fn get_eks_a_bundle_url<S>(
eks_a_release_manifest_url: S,
resources: &Resources,
) -> ProviderResult<String>
where
S: AsRef<str> + IntoUrl + Display,
{
let deserialized_manifest = get_eks_a_manifest(eks_a_release_manifest_url, resources).await?;

let latest_release_version = deserialized_manifest
.iter()
.find(|config| {
config.get("kind") == Some(&serde_yaml::Value::String("Release".to_string()))
})
.and_then(|release| release.get("spec"))
.and_then(|spec| spec.get("latestVersion"))
.and_then(|ver| ver.as_str())
.context(resources, "Unable to get latest version for EKS-A")?;

deserialized_manifest
.iter()
.find(|manifests| {
manifests.get("kind") == Some(&serde_yaml::Value::String("Release".to_string()))
})
.and_then(|manifest| manifest.get("spec"))
.and_then(|spec| spec.get("releases"))
.and_then(|list| list.as_sequence())
.and_then(|releases| {
releases.iter().find(|release| {
release.get("version")
== Some(&serde_yaml::Value::String(
latest_release_version.to_string(),
))
})
})
.and_then(|release| release.get("bundleManifestUrl"))
.and_then(|uri| uri.as_str())
.context(
resources,
format!(
"Unable to get the bundle URL for the latest EKS-A version ({})",
latest_release_version
),
)
.map(|s| s.to_string())
}

pub async fn fetch_eks_a_binary(
archive_url: String,
dest: PathBuf,
Expand Down Expand Up @@ -196,3 +221,71 @@ pub async fn install_eks_a_binary(
info!("Fetching EKS-A binary archive from '{}'", eks_a_archive_url);
fetch_eks_a_binary(eks_a_archive_url, PathBuf::from(USR_LOCAL_BIN), resources).await
}

pub async fn download_eks_a_bundle(
eks_a_release_manifest_url: &Option<String>,
bundle_manifest_path: &str,
resources: &Resources,
) -> ProviderResult<()> {
let eks_a_release_manifest_url = eks_a_release_manifest_url
.to_owned()
.unwrap_or(DEFAULT_EKS_A_RELEASE_MANIFEST.to_string());
info!(
"Using EKS-A release manifest '{}'",
eks_a_release_manifest_url
);
let eks_a_bundle_url = get_eks_a_bundle_url(eks_a_release_manifest_url, resources).await?;
info!(
"Downloading EKS-A bundle manifest file from '{}'",
eks_a_bundle_url
);
let mut bundle_manifest_request = reqwest::blocking::get(eks_a_bundle_url.as_str())
.context(resources, "Could not send HTTP GET request for bundle.yaml")?
.error_for_status()
.context(
Resources::Clear,
"Bad HTTP response when downloading bundle.yaml",
)?;
let mut f = File::create(bundle_manifest_path)
.context(resources, "Failed to create bundle.yaml file")?;
bundle_manifest_request
.copy_to(&mut f)
.context(Resources::Clear, "Failed to copy bundle.yaml to file")?;

Ok(())
}

async fn get_eks_a_manifest<S>(
eks_a_release_manifest_url: S,
resources: &Resources,
) -> ProviderResult<Vec<serde_yaml::Value>>
where
S: AsRef<str> + IntoUrl + Display,
{
let manifest = reqwest::get(eks_a_release_manifest_url.to_string())
.await
.context(
resources,
format!(
"Unable to request EKS-A release manifest '{}'",
eks_a_release_manifest_url
),
)?
.text()
.await
.context(
resources,
format!(
"Unable to retrieve EKS-A release manifest at '{}'",
eks_a_release_manifest_url
),
)?;
let deserialized_manifest = serde_yaml::Deserializer::from_str(&manifest)
.map(|config| {
serde_yaml::Value::deserialize(config)
.context(resources, "Unable to deserialize eksa config file")
})
.collect::<ProviderResult<Vec<_>>>()?;

Ok(deserialized_manifest)
}
Loading