diff --git a/Cargo.lock b/Cargo.lock index b22430c..25c8d18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2371,6 +2371,37 @@ dependencies = [ "gadget-std", ] +[[package]] +name = "blueprint-manager" +version = "0.2.2" +dependencies = [ + "async-trait", + "auto_impl", + "clap", + "color-eyre", + "futures", + "gadget-clients", + "gadget-config", + "gadget-crypto", + "gadget-keystore", + "gadget-logging", + "gadget-networking", + "gadget-std", + "hex", + "itertools 0.13.0", + "libp2p", + "parking_lot", + "reqwest 0.12.9", + "serde", + "sha2 0.10.8", + "tangle-subxt", + "thiserror 2.0.7", + "tokio", + "toml", + "tracing", + "tracing-subscriber 0.3.19", +] + [[package]] name = "blueprint-metadata" version = "0.2.1" @@ -2390,6 +2421,7 @@ name = "blueprint-util-meta" version = "0.1.0" dependencies = [ "blueprint-build-utils", + "blueprint-manager", "blueprint-metadata", ] @@ -2943,10 +2975,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5" dependencies = [ "backtrace", + "color-spantrace", "eyre", "indenter", "once_cell", "owo-colors", + "tracing-error", + "url", +] + +[[package]] +name = "color-spantrace" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", ] [[package]] @@ -5064,6 +5111,7 @@ dependencies = [ "gadget-client-evm", "gadget-client-networking", "gadget-client-tangle", + "thiserror 2.0.7", ] [[package]] @@ -12790,6 +12838,16 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-error" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +dependencies = [ + "tracing", + "tracing-subscriber 0.3.19", +] + [[package]] name = "tracing-futures" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 40477ee..b1c1e57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ broken_intra_doc_links = "deny" [workspace.dependencies] # Blueprint utils blueprint-util-meta = { version = "0.1.0", path = "./crates/blueprint", default-features = false } +blueprint-manager = { version = "0.2.2", path = "./crates/blueprint/manager", default-features = false } blueprint-metadata = { version = "0.2.1", path = "./crates/blueprint/metadata", default-features = false } blueprint-build-utils = { version = "0.1.0", path = "./crates/blueprint/build-utils", default-features = false } gadget-blueprint-serde = { version = "0.3.1", path = "./crates/blueprint/serde", default-features = false } diff --git a/cli/src/deploy/tangle.rs b/cli/src/deploy/tangle.rs index 7f38ada..280c946 100644 --- a/cli/src/deploy/tangle.rs +++ b/cli/src/deploy/tangle.rs @@ -4,17 +4,15 @@ use alloy_rpc_types_eth::TransactionRequest; use alloy_signer_local::PrivateKeySigner; use color_eyre::eyre::{self, Context, ContextCompat, Result}; use gadget_blueprint_proc_macro_core::{BlueprintManager, ServiceBlueprint}; -use gadget_clients::tangle::runtime::TangleConfig; +use gadget_crypto::tangle_pair_signer::TanglePairSigner; use gadget_std::fmt::Debug; use gadget_std::path::PathBuf; +use subxt::tx::Signer; use tangle_subxt::subxt; use tangle_subxt::subxt::ext::sp_core; -use tangle_subxt::subxt::tx::PairSigner; use tangle_subxt::tangle_testnet_runtime::api as TangleApi; use tangle_subxt::tangle_testnet_runtime::api::services::calls::types; -pub type TanglePairSigner = PairSigner; - #[derive(Clone)] pub struct Opts { /// The name of the package to deploy (if the workspace has multiple packages) @@ -26,7 +24,7 @@ pub struct Opts { /// The path to the manifest file pub manifest_path: gadget_std::path::PathBuf, /// The signer for deploying the blueprint - pub signer: Option, + pub signer: Option>, /// The signer for deploying the smart contract pub signer_evm: Option, } @@ -106,7 +104,7 @@ pub async fn deploy_to_tangle( let signer = if let Some(signer) = signer { signer } else { - crate::signer::load_signer_from_env()?.pair + crate::signer::load_signer_from_env()? }; let my_account_id = signer.account_id(); diff --git a/crates/blueprint/Cargo.toml b/crates/blueprint/Cargo.toml index ef09d1c..c5c9aca 100644 --- a/crates/blueprint/Cargo.toml +++ b/crates/blueprint/Cargo.toml @@ -9,5 +9,6 @@ repository.workspace = true publish = false [dependencies] +blueprint-manager.workspace = true blueprint-metadata.workspace = true blueprint-build-utils.workspace = true \ No newline at end of file diff --git a/crates/blueprint/manager/CHANGELOG.md b/crates/blueprint/manager/CHANGELOG.md new file mode 100644 index 0000000..7145f63 --- /dev/null +++ b/crates/blueprint/manager/CHANGELOG.md @@ -0,0 +1,87 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.2.2](https://github.com/tangle-network/gadget/compare/blueprint-manager-v0.2.1...blueprint-manager-v0.2.2) - 2024-12-11 + +### Other + +- Call ID Insertion and Resolution + For [#520](https://github.com/tangle-network/gadget/pull/520) ([#533](https://github.com/tangle-network/gadget/pull/533)) + +## [0.2.1](https://github.com/tangle-network/gadget/compare/blueprint-manager-v0.2.0...blueprint-manager-v0.2.1) - 2024-12-04 + +### Other + +- updated the following local packages: gadget-sdk + +## [0.2.0](https://github.com/tangle-network/gadget/compare/blueprint-manager-v0.1.3...blueprint-manager-v0.2.0) - 2024-11-29 + +### Other + +- *(gadget-sdk)* [**breaking**] update to latest tangle ([#503](https://github.com/tangle-network/gadget/pull/503)) + +## [0.1.3](https://github.com/tangle-network/gadget/compare/blueprint-manager-v0.1.2...blueprint-manager-v0.1.3) - 2024-11-20 + +### Other + +- updated the following local packages: gadget-sdk + +## [0.1.2](https://github.com/tangle-network/gadget/compare/blueprint-manager-v0.1.1...blueprint-manager-v0.1.2) - 2024-11-16 + +### Other + +- updated the following local packages: gadget-sdk + +## [0.1.1](https://github.com/tangle-network/gadget/releases/tag/blueprint-manager-v0.1.1) - 2024-11-08 + +### Added + +- [**breaking**] Refactor EventFlows for EVM and Remove + EventWatchers ([#423](https://github.com/tangle-network/gadget/pull/423)) +- symbiotic initial integration ([#411](https://github.com/tangle-network/gadget/pull/411)) +- add optional data dir to blueprint manager ([#342](https://github.com/tangle-network/gadget/pull/342)) +- eigenlayer incredible squaring blueprint and test ([#312](https://github.com/tangle-network/gadget/pull/312)) +- add EVM Provider and Tangle Client Context Extensions ([#319](https://github.com/tangle-network/gadget/pull/319)) +- Keystore Context Extensions ([#316](https://github.com/tangle-network/gadget/pull/316)) +- add benchmarking mode ([#248](https://github.com/tangle-network/gadget/pull/248)) + +### Fixed + +- *(gadget-sdk)* [**breaking**] prevent duplicate and self-referential + messages ([#458](https://github.com/tangle-network/gadget/pull/458)) +- *(sdk)* [**breaking**] allow for zero-based `blueprint_id` ([#426](https://github.com/tangle-network/gadget/pull/426)) +- *(cargo-tangle)* CLI bugs ([#409](https://github.com/tangle-network/gadget/pull/409)) +- *(sdk)* [**breaking**] downgrade substrate dependencies for now +- add `data_dir` back to `GadgetConfiguration` ([#350](https://github.com/tangle-network/gadget/pull/350)) + +### Other + +- set blueprint-manager publishable ([#462](https://github.com/tangle-network/gadget/pull/462)) +- add a p2p test for testing the networking layer ([#450](https://github.com/tangle-network/gadget/pull/450)) +- Continue Improving Event Flows ([#399](https://github.com/tangle-network/gadget/pull/399)) +- improve blueprint-manager and blueprint-test-utils ([#421](https://github.com/tangle-network/gadget/pull/421)) +- Leverage blueprint in incredible squaring aggregator ([#365](https://github.com/tangle-network/gadget/pull/365)) +- Event Listener Upgrade + Wrapper Types + sdk::main macro ([#333](https://github.com/tangle-network/gadget/pull/333)) +- docs fix spelling issues ([#336](https://github.com/tangle-network/gadget/pull/336)) +- Event listener ([#317](https://github.com/tangle-network/gadget/pull/317)) +- Remove Logger ([#311](https://github.com/tangle-network/gadget/pull/311)) +- Streamline keystore, cleanup testing, refactor blueprint manager, add tests, remove unnecessary + code ([#285](https://github.com/tangle-network/gadget/pull/285)) +- CI Improvements ([#301](https://github.com/tangle-network/gadget/pull/301)) +- [feat] Gadget Metadata ([#274](https://github.com/tangle-network/gadget/pull/274)) +- [MEGA PR] Overhaul repo, add Eigenlayer AVS example, remove many crates, add testing, remove unused + code ([#246](https://github.com/tangle-network/gadget/pull/246)) +- Add mpc blueprint starting point, cleanup abstractions ([#252](https://github.com/tangle-network/gadget/pull/252)) +- Add more checks to CI ([#244](https://github.com/tangle-network/gadget/pull/244)) +- [feat] benchmark proc-macro ([#238](https://github.com/tangle-network/gadget/pull/238)) +- Spelling fix +- Promote all dependencies to workspace ([#233](https://github.com/tangle-network/gadget/pull/233)) +- Make `{core, io, common}` no_std and WASM compatible ([#231](https://github.com/tangle-network/gadget/pull/231)) +- Remove shell sdk and put inside blueprint manager ([#229](https://github.com/tangle-network/gadget/pull/229)) +- Blueprint testing ([#206](https://github.com/tangle-network/gadget/pull/206)) diff --git a/crates/blueprint/manager/Cargo.toml b/crates/blueprint/manager/Cargo.toml new file mode 100644 index 0000000..8fe8e0e --- /dev/null +++ b/crates/blueprint/manager/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "blueprint-manager" +version = "0.2.2" +description = "Tangle Blueprint manager and Runner" +authors.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true + +[[bin]] +name = "blueprint-manager" +path = "src/main.rs" + +[dependencies] +gadget-clients = { workspace = true, features = ["std", "tangle"] } +gadget-config = { workspace = true, features = ["std", "networking"] } +gadget-crypto = { workspace = true, features = ["std", "tangle-pair-signer"] } +gadget-keystore = { workspace = true, features = ["std", "tangle"] } +gadget-logging = { workspace = true, features = ["std"] } +gadget-networking = { workspace = true, features = ["std"] } +gadget-std = { workspace = true, features = ["std"] } + +clap = { workspace = true, features = ["derive", "wrap_help"] } +color-eyre = { workspace = true, features = ["tracing-error", "color-spantrace", "issue-url"] } +serde = { workspace = true } +tangle-subxt = { workspace = true } +toml = { workspace = true } +hex = { workspace = true } +tokio = { workspace = true, features = ["process", "io-util", "signal", "macros"] } +reqwest = { workspace = true } +sha2 = { workspace = true } +futures = { workspace = true } +itertools = { workspace = true } +thiserror.workspace = true +tracing = { workspace = true, features = ["log"] } +tracing-subscriber = { workspace = true, features = ["env-filter", "ansi", "tracing-log"] } +libp2p = { workspace = true } +auto_impl = { workspace = true } +parking_lot = { workspace = true } +async-trait = { workspace = true } + +[lints] +workspace = true + +[package.metadata.dist] +dist = false diff --git a/crates/blueprint/manager/src/config.rs b/crates/blueprint/manager/src/config.rs new file mode 100644 index 0000000..207427f --- /dev/null +++ b/crates/blueprint/manager/src/config.rs @@ -0,0 +1,31 @@ +use clap::Parser; +use std::path::PathBuf; + +#[derive(Debug, Parser)] +#[command( + name = "Blueprint Manager", + about = "An program executor that connects to the Tangle network and runs protocols dynamically on the fly" +)] +pub struct BlueprintManagerConfig { + /// The path to the gadget configuration file + #[arg(short = 's', long)] + pub gadget_config: Option, + /// The path to the keystore + #[arg(short = 'k', long)] + pub keystore_uri: String, + /// The directory in which all gadgets will store their data + #[arg(long, short = 'd', default_value = "./data")] + pub data_dir: PathBuf, + /// The verbosity level, can be used multiple times to increase verbosity + #[arg(long, short = 'v', action = clap::ArgAction::Count)] + pub verbose: u8, + /// Whether to use pretty logging + #[arg(long)] + pub pretty: bool, + /// An optional unique string identifier for the blueprint manager to differentiate between multiple + /// running instances of a BlueprintManager (mostly for debugging purposes) + #[arg(long, alias = "id")] + pub instance_id: Option, + #[arg(long, short = 't')] + pub test_mode: bool, +} diff --git a/crates/blueprint/manager/src/error.rs b/crates/blueprint/manager/src/error.rs new file mode 100644 index 0000000..01ac696 --- /dev/null +++ b/crates/blueprint/manager/src/error.rs @@ -0,0 +1,39 @@ +pub type Result = std::result::Result; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("No fetchers found for blueprint")] + NoFetchers, + #[error("Multiple fetchers found for blueprint")] + MultipleFetchers, + #[error("No testing fetcher found for blueprint, despite operating in test mode")] + NoTestFetcher, + #[error("Blueprint does not contain a supported fetcher")] + UnsupportedGadget, + + #[error("Unable to find matching binary")] + NoMatchingBinary, + #[error("Binary hash {expected} mismatched expected hash of {actual}")] + HashMismatch { expected: String, actual: String }, + #[error("Failed to build binary: {0:?}")] + BuildBinary(std::process::Output), + #[error("Failed to fetch git root: {0:?}")] + FetchGitRoot(std::process::Output), + + #[error("Failed to get initial block hash")] + InitialBlock, + #[error("Finality Notification stream died")] + ClientDied, + #[error("{0}")] + Other(String), + + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Utf8(#[from] std::string::FromUtf8Error), + + #[error(transparent)] + Request(#[from] reqwest::Error), + #[error(transparent)] + TangleClient(#[from] gadget_clients::tangle::error::Error), +} diff --git a/crates/blueprint/manager/src/executor/event_handler.rs b/crates/blueprint/manager/src/executor/event_handler.rs new file mode 100644 index 0000000..5e01e22 --- /dev/null +++ b/crates/blueprint/manager/src/executor/event_handler.rs @@ -0,0 +1,448 @@ +use crate::config::BlueprintManagerConfig; +use crate::error::{Error, Result}; +use crate::gadget::native::FilteredBlueprint; +use crate::gadget::ActiveGadgets; +use crate::sdk::utils::{ + bounded_string_to_string, generate_running_process_status_handle, make_executable, +}; +use crate::sources::github::GithubBinaryFetcher; +use crate::sources::{process_arguments_and_env, BinarySourceFetcher}; +use gadget_clients::tangle::client::{TangleConfig, TangleEvent}; +use gadget_clients::tangle::services::{RpcServicesWithBlueprint, TangleServicesClient}; +use gadget_config::{GadgetConfiguration, Protocol}; +use gadget_logging::{error, info, trace, warn}; +use std::fmt::Debug; +use std::sync::atomic::Ordering; +use tangle_subxt::subxt::utils::AccountId32; +use tangle_subxt::tangle_testnet_runtime::api::runtime_types::tangle_primitives::services::{ + Gadget, GadgetSourceFetcher, +}; +use tangle_subxt::tangle_testnet_runtime::api::services::events::{ + JobCalled, JobResultSubmitted, PreRegistration, Registered, ServiceInitiated, Unregistered, +}; + +pub struct VerifiedBlueprint<'a> { + pub(crate) fetcher: Box, + pub(crate) blueprint: FilteredBlueprint, +} + +impl VerifiedBlueprint<'_> { + pub async fn start_services_if_needed( + &self, + gadget_config: &GadgetConfiguration, + blueprint_manager_opts: &BlueprintManagerConfig, + active_gadgets: &mut ActiveGadgets, + ) -> Result<()> { + let blueprint_source = &self.fetcher; + let blueprint = &self.blueprint; + + let blueprint_id = blueprint_source.blueprint_id(); + if active_gadgets.contains_key(&blueprint_id) { + return Ok(()); + } + + let mut binary_download_path = blueprint_source.get_binary().await?; + + // Ensure the binary is executable + if cfg!(target_family = "windows") { + if binary_download_path.extension().is_none() { + binary_download_path.set_extension("exe"); + } + } else if let Err(err) = make_executable(&binary_download_path) { + let msg = format!("Failed to make the binary executable: {err}"); + warn!("{}", msg); + return Err(Error::Other(msg)); + } + + let service_str = blueprint_source.name(); + for service_id in &blueprint.services { + let sub_service_str = format!("{service_str}-{service_id}"); + let (arguments, env_vars) = process_arguments_and_env( + gadget_config, + blueprint_manager_opts, + blueprint_id, + *service_id, + blueprint, + &sub_service_str, + ); + + info!("Starting protocol: {sub_service_str} with args: {arguments:?}"); + + // Now that the file is loaded, spawn the process + let process_handle = tokio::process::Command::new(&binary_download_path) + .kill_on_drop(true) + .stdin(std::process::Stdio::null()) + .current_dir(&std::env::current_dir()?) + .envs(env_vars) + .args(arguments) + .spawn()?; + + if blueprint.registration_mode { + // We must wait for the process to exit successfully + let status = process_handle.wait_with_output().await?; + if status.status.success() { + info!("***Protocol (registration mode) {sub_service_str} executed successfully***"); + } else { + error!( + "Protocol (registration mode) {sub_service_str} failed to execute: {status:?}" + ); + } + continue; + } + + // A normal running gadget binary. Store the process handle and let the event loop handle the rest + + let (status_handle, abort) = + generate_running_process_status_handle(process_handle, &sub_service_str); + + active_gadgets + .entry(blueprint_id) + .or_default() + .insert(*service_id, (status_handle, Some(abort))); + } + + Ok(()) + } +} + +impl Debug for VerifiedBlueprint<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + format!( + "{}/bid={}/sid(s)={:?}", + self.blueprint.name, self.blueprint.blueprint_id, self.blueprint.services + ) + .fmt(f) + } +} + +#[derive(Default, Debug)] +pub struct EventPollResult { + pub needs_update: bool, + // A vec of blueprints we have not yet become registered to + pub blueprint_registrations: Vec, +} + +pub(crate) fn check_blueprint_events( + event: &TangleEvent, + active_gadgets: &mut ActiveGadgets, + account_id: &AccountId32, +) -> EventPollResult { + let pre_registation_events = event.events.find::(); + let registered_events = event.events.find::(); + let unregistered_events = event.events.find::(); + let service_initiated_events = event.events.find::(); + let job_called_events = event.events.find::(); + let job_result_submitted_events = event.events.find::(); + + let mut result = EventPollResult::default(); + + for evt in pre_registation_events { + match evt { + Ok(evt) => { + if &evt.operator == account_id { + result.blueprint_registrations.push(evt.blueprint_id); + info!("Pre-registered event: {evt:?}"); + } + } + Err(err) => { + warn!("Error handling pre-registered event: {err:?}"); + } + } + } + + // Handle registered events + for evt in registered_events { + match evt { + Ok(evt) => { + info!("Registered event: {evt:?}"); + result.needs_update = true; + } + Err(err) => { + warn!("Error handling registered event: {err:?}"); + } + } + } + + // Handle unregistered events + for evt in unregistered_events { + match evt { + Ok(evt) => { + info!("Unregistered event: {evt:?}"); + if &evt.operator == account_id && active_gadgets.remove(&evt.blueprint_id).is_some() + { + info!("Removed services for blueprint_id: {}", evt.blueprint_id,); + + result.needs_update = true; + } + } + Err(err) => { + warn!("Error handling unregistered event: {err:?}"); + } + } + } + + // Handle service initiated events + for evt in service_initiated_events { + match evt { + Ok(evt) => { + info!("Service initiated event: {evt:?}"); + } + Err(err) => { + warn!("Error handling service initiated event: {err:?}"); + } + } + } + + // Handle job called events + for evt in job_called_events { + match evt { + Ok(evt) => { + info!("Job called event: {evt:?}"); + } + Err(err) => { + warn!("Error handling job called event: {err:?}"); + } + } + } + + // Handle job result submitted events + for evt in job_result_submitted_events { + match evt { + Ok(evt) => { + info!("Job result submitted event: {evt:?}"); + } + Err(err) => { + warn!("Error handling job result submitted event: {err:?}"); + } + } + } + + result +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn handle_tangle_event( + event: &TangleEvent, + blueprints: &[RpcServicesWithBlueprint], + gadget_config: &GadgetConfiguration, + manager_opts: &BlueprintManagerConfig, + active_gadgets: &mut ActiveGadgets, + poll_result: EventPollResult, + client: &TangleServicesClient, +) -> Result<()> { + info!("Received notification {}", event.number); + const DEFAULT_PROTOCOL: Protocol = Protocol::Tangle; + warn!("Using Tangle protocol as default over Eigen. This is a temporary development workaround. You can alter this behavior here"); + + // const DEFAULT_PROTOCOL: Protocol = Protocol::Eigenlayer; + // warn!("Using Eigen protocol as default over Tangle. This is a temporary development workaround. You can alter this behavior here"); + + let mut registration_blueprints = vec![]; + // First, check to see if we need to register any new services invoked by the PreRegistration event + if !poll_result.blueprint_registrations.is_empty() { + for blueprint_id in &poll_result.blueprint_registrations { + let blueprint = client + .get_blueprint_by_id(event.hash, *blueprint_id) + .await? + .ok_or_else(|| { + Error::Other(String::from( + "Unable to retrieve blueprint for registration mode", + )) + })?; + + let general_blueprint = FilteredBlueprint { + blueprint_id: *blueprint_id, + services: vec![0], // Add a dummy service id for now, since it does not matter for registration mode + gadget: blueprint.gadget, + name: bounded_string_to_string(blueprint.metadata.name)?, + registration_mode: true, + protocol: DEFAULT_PROTOCOL, + }; + + registration_blueprints.push(general_blueprint); + } + } + + let mut verified_blueprints = vec![]; + + for blueprint in blueprints + .iter() + .map(|r| FilteredBlueprint { + blueprint_id: r.blueprint_id, + services: r.services.iter().map(|r| r.id).collect(), + gadget: r.blueprint.gadget.clone(), + name: bounded_string_to_string(r.clone().blueprint.metadata.name) + .unwrap_or("unknown_blueprint_name".to_string()), + registration_mode: false, + protocol: DEFAULT_PROTOCOL, + }) + .chain(registration_blueprints) + { + let mut fetcher_candidates = get_fetcher_candidates(&blueprint, manager_opts)?; + + let verified_blueprint = VerifiedBlueprint { + fetcher: fetcher_candidates.pop().expect("Should exist"), + blueprint, + }; + + verified_blueprints.push(verified_blueprint); + } + + trace!( + "OnChain Verified Blueprints: {:?}", + verified_blueprints + .iter() + .map(|r| format!("{r:?}")) + .collect::>() + ); + + // Step 3: Check to see if we need to start any new services + for blueprint in &verified_blueprints { + blueprint + .start_services_if_needed(gadget_config, manager_opts, active_gadgets) + .await?; + } + + // Check to see if local is running services that are not on-chain + let mut to_remove: Vec<(u64, u64)> = vec![]; + + // Loop through every (blueprint_id, service_id) running. See if the service is still on-chain. If not, kill it and add it to to_remove + for (blueprint_id, process_handles) in &mut *active_gadgets { + for (service_id, process_handle) in process_handles { + info!( + "Checking service for on-chain termination: bid={blueprint_id}//sid={service_id}" + ); + + // Since the below "verified blueprints" were freshly obtained from an on-chain source, + // we compare all these fresh values to see if we're running a service locally that is no longer on-chain + for verified_blueprints in &verified_blueprints { + let services = &verified_blueprints.blueprint.services; + // Safe assertion since we know there is at least one fetcher. All fetchers should have the same blueprint id + let fetcher = &verified_blueprints.fetcher; + if fetcher.blueprint_id() == *blueprint_id && !services.contains(service_id) { + warn!("Killing service that is no longer on-chain: bid={blueprint_id}//sid={service_id}"); + to_remove.push((*blueprint_id, *service_id)); + } + } + + // Check to see if any process handles have died + if !to_remove.contains(&(*blueprint_id, *service_id)) + && !process_handle.0.load(Ordering::Relaxed) + { + // By removing any killed processes, we will auto-restart them on the next finality notification if required + warn!("Killing service that has died to allow for auto-restart"); + to_remove.push((*blueprint_id, *service_id)); + } + } + } + + for (blueprint_id, service_id) in to_remove { + warn!("Removing service that is no longer active on-chain or killed: bid={blueprint_id}//sid={service_id}"); + let mut should_delete_blueprint = false; + if let Some(gadgets) = active_gadgets.get_mut(&blueprint_id) { + if let Some((_, mut process_handle)) = gadgets.remove(&service_id) { + if let Some(abort_handle) = process_handle.take() { + if abort_handle.send(()).is_err() { + error!("Failed to send abort signal to service: bid={blueprint_id}//sid={service_id}"); + } else { + warn!("Sent abort signal to service: bid={blueprint_id}//sid={service_id}"); + } + } + } + + if gadgets.is_empty() { + should_delete_blueprint = true; + } + } + + if should_delete_blueprint { + active_gadgets.remove(&blueprint_id); + } + } + + Ok(()) +} + +fn get_fetcher_candidates( + blueprint: &FilteredBlueprint, + manager_opts: &BlueprintManagerConfig, +) -> Result>> { + let mut test_fetcher_idx = None; + let mut fetcher_candidates: Vec> = vec![]; + + let sources; + match &blueprint.gadget { + Gadget::Native(gadget) => { + sources = &gadget.sources.0; + } + Gadget::Wasm(_) => { + warn!("WASM gadgets are not supported yet"); + return Err(Error::UnsupportedGadget); + } + Gadget::Container(_) => { + warn!("Container gadgets are not supported yet"); + return Err(Error::UnsupportedGadget); + } + } + + for (source_idx, gadget_source) in sources.iter().enumerate() { + match &gadget_source.fetcher { + GadgetSourceFetcher::Github(gh) => { + let fetcher = GithubBinaryFetcher { + fetcher: gh.clone(), + blueprint_id: blueprint.blueprint_id, + gadget_name: blueprint.name.clone(), + }; + + fetcher_candidates.push(Box::new(fetcher)); + } + + GadgetSourceFetcher::Testing(test) => { + // TODO: demote to TRACE once proven to work + if !manager_opts.test_mode { + warn!("Ignoring testing fetcher as we are not in test mode"); + continue; + } + + let fetcher = crate::sources::testing::TestSourceFetcher { + fetcher: test.clone(), + blueprint_id: blueprint.blueprint_id, + gadget_name: blueprint.name.clone(), + }; + + test_fetcher_idx = Some(source_idx); + fetcher_candidates.push(Box::new(fetcher)); + } + + _ => { + warn!("Blueprint does not contain a supported fetcher"); + continue; + } + } + } + + // A bunch of sanity checks to enforce structure + + // Ensure that we have at least one fetcher + if fetcher_candidates.is_empty() { + return Err(Error::NoFetchers); + } + + // Ensure there is only a single candidate fetcher + if fetcher_candidates.len() != 1 { + return Err(Error::MultipleFetchers); + } + + // Ensure that we have a test fetcher if we are in test mode + if manager_opts.test_mode && test_fetcher_idx.is_none() { + return Err(Error::NoTestFetcher); + } + + // Ensure that we have only one fetcher if we are in test mode + if manager_opts.test_mode { + fetcher_candidates = + vec![fetcher_candidates.remove(test_fetcher_idx.expect("Should exist"))]; + } + + Ok(fetcher_candidates) +} diff --git a/crates/blueprint/manager/src/executor/mod.rs b/crates/blueprint/manager/src/executor/mod.rs new file mode 100644 index 0000000..118ac5a --- /dev/null +++ b/crates/blueprint/manager/src/executor/mod.rs @@ -0,0 +1,310 @@ +use crate::config::BlueprintManagerConfig; +use crate::error::Error; +use crate::error::Result; +use crate::gadget::ActiveGadgets; +use crate::sdk::entry::SendFuture; +use color_eyre::eyre::OptionExt; +use color_eyre::Report; +use gadget_clients::tangle::client::{TangleClient, TangleConfig}; +use gadget_clients::tangle::services::{RpcServicesWithBlueprint, TangleServicesClient}; +use gadget_clients::tangle::EventsClient; +use gadget_config::GadgetConfiguration; +use gadget_crypto::tangle_pair_signer::TanglePairSigner; +use gadget_keystore::backends::tangle::TangleBackend; +use gadget_keystore::{Keystore, KeystoreConfig}; +use gadget_logging::info; +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tangle_subxt::subxt::ext::sp_core::{ecdsa, sr25519}; +use tangle_subxt::subxt::tx::Signer; +use tangle_subxt::subxt::utils::AccountId32; +use tokio::task::JoinHandle; + +pub(crate) mod event_handler; + +pub struct BlueprintManagerHandle { + shutdown_call: Option>, + start_tx: Option>, + running_task: JoinHandle>, + span: tracing::Span, + sr25519_id: TanglePairSigner, + ecdsa_id: TanglePairSigner, + keystore_uri: String, +} + +impl BlueprintManagerHandle { + /// Send a start signal to the blueprint manager + pub fn start(&mut self) -> color_eyre::Result<()> { + let _span = self.span.enter(); + match self.start_tx.take() { + Some(tx) => match tx.send(()) { + Ok(_) => { + info!("Start signal sent to Blueprint Manager"); + Ok(()) + } + Err(_) => Err(Report::msg( + "Failed to send start signal to Blueprint Manager", + )), + }, + None => Err(Report::msg("Blueprint Manager Already Started")), + } + } + + /// Returns the SR25519 keypair for this blueprint manager + pub fn sr25519_id(&self) -> &TanglePairSigner { + &self.sr25519_id + } + + /// Returns the ECDSA keypair for this blueprint manager + pub fn ecdsa_id(&self) -> &TanglePairSigner { + &self.ecdsa_id + } + + /// Shutdown the blueprint manager + pub async fn shutdown(&mut self) -> color_eyre::Result<()> { + self.shutdown_call + .take() + .map(|tx| tx.send(())) + .ok_or_eyre("Shutdown already called")? + .map_err(|_| Report::msg("Failed to send shutdown signal to Blueprint Manager")) + } + + /// Returns the keystore URI for this blueprint manager + pub fn keystore_uri(&self) -> &str { + &self.keystore_uri + } + + pub fn span(&self) -> &tracing::Span { + &self.span + } +} + +/// Add default behavior for unintentional dropping of the BlueprintManagerHandle +/// This will ensure that the BlueprintManagerHandle is executed even if the handle +/// is dropped, which is similar behavior to the tokio SpawnHandle +impl Drop for BlueprintManagerHandle { + fn drop(&mut self) { + let _ = self.start(); + } +} + +/// Implement the Future trait for the BlueprintManagerHandle to allow +/// for the handle to be awaited on +impl Future for BlueprintManagerHandle { + type Output = color_eyre::Result<()>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // Start the blueprint manager if it has not been started + let this = self.get_mut(); + if this.start_tx.is_some() { + if let Err(err) = this.start() { + return Poll::Ready(Err(err)); + } + } + + let result = futures::ready!(Pin::new(&mut this.running_task).poll(cx)); + + match result { + Ok(res) => Poll::Ready(res), + Err(err) => Poll::Ready(Err(Report::msg(format!( + "Blueprint Manager Closed Unexpectedly (JoinError): {err:?}" + )))), + } + } +} + +#[allow(clippy::too_many_lines)] +pub async fn run_blueprint_manager>( + blueprint_manager_config: BlueprintManagerConfig, + gadget_config: GadgetConfiguration, + shutdown_cmd: F, +) -> color_eyre::Result { + let logger_id = if let Some(custom_id) = &blueprint_manager_config.instance_id { + custom_id.as_str() + } else { + "Local" + }; + + let span = tracing::info_span!("Blueprint-Manager", id = logger_id); + + let _span = span.enter(); + info!("Starting blueprint manager ... waiting for start signal ..."); + + let data_dir = &blueprint_manager_config.data_dir; + if !data_dir.exists() { + info!( + "Data directory does not exist, creating it at `{}`", + data_dir.display() + ); + std::fs::create_dir_all(data_dir)?; + } + + // TODO: Actual error handling + let (tangle_key, ecdsa_key) = { + let keystore = Keystore::new(KeystoreConfig::new().fs_root(&gadget_config.keystore_uri))?; + let sr_key_pub = keystore + .iter_sr25519() + .next() + .expect("No SR25519 keys found"); + let sr_pair = keystore + .expose_sr25519_secret(&sr_key_pub)? + .expect("No matching SR25519 key"); + let sr_key = TanglePairSigner::new(sr_pair); + + let ecdsa_key_pub = keystore.iter_ecdsa().next().expect("No ECDSA keys found"); + let ecdsa_pair = keystore + .expose_ecdsa_secret(&ecdsa_key_pub)? + .expect("No matching ECDSA key"); + let ecdsa_key = TanglePairSigner::new(ecdsa_pair); + + (sr_key, ecdsa_key) + }; + + let sub_account_id = tangle_key.account_id().clone(); + + let mut active_gadgets = HashMap::new(); + + let keystore_uri = gadget_config.keystore_uri.clone(); + + let manager_task = async move { + let tangle_client = TangleClient::new(gadget_config.clone()).await?; + let services_client = tangle_client.services_client(); + + // With the basics setup, we must now implement the main logic of the Blueprint Manager + // Handle initialization logic + // NOTE: The node running this code should be registered as an operator for the blueprints, otherwise, this + // code will fail + let mut operator_subscribed_blueprints = handle_init( + &tangle_client, + services_client, + &sub_account_id, + &mut active_gadgets, + &gadget_config, + &blueprint_manager_config, + ) + .await?; + + // Now, run the main event loop + // Listen to FinalityNotifications and poll for new/deleted services that correspond to the blueprints above + while let Some(event) = tangle_client.next_event().await { + let result = event_handler::check_blueprint_events( + &event, + &mut active_gadgets, + &sub_account_id.clone(), + ); + + if result.needs_update { + operator_subscribed_blueprints = services_client + .query_operator_blueprints(event.hash, sub_account_id.clone()) + .await?; + } + + event_handler::handle_tangle_event( + &event, + &operator_subscribed_blueprints, + &gadget_config, + &blueprint_manager_config, + &mut active_gadgets, + result, + services_client, + ) + .await?; + } + + Err::<(), _>(Error::ClientDied) + }; + + let (tx_stop, rx_stop) = tokio::sync::oneshot::channel::<()>(); + + let shutdown_task = async move { + tokio::select! { + _res0 = shutdown_cmd => { + info!("Shutdown-1 command received, closing application"); + }, + + _res1 = rx_stop => { + info!("Manual shutdown signal received, closing application"); + } + } + }; + + let (start_tx, start_rx) = tokio::sync::oneshot::channel::<()>(); + + let combined_task = async move { + start_rx + .await + .map_err(|_err| Report::msg("Failed to receive start signal"))?; + + tokio::select! { + res0 = manager_task => { + Err(Report::msg(format!("Blueprint Manager Closed Unexpectedly: {res0:?}"))) + }, + + _ = shutdown_task => { + Ok(()) + } + } + }; + + drop(_span); + let handle = tokio::spawn(combined_task); + + let handle = BlueprintManagerHandle { + start_tx: Some(start_tx), + shutdown_call: Some(tx_stop), + running_task: handle, + span, + sr25519_id: tangle_key, + ecdsa_id: ecdsa_key, + keystore_uri, + }; + + Ok(handle) +} + +/// * Query to get Vec +/// * For each RpcServicesWithBlueprint, fetch the associated gadget binary (fetch/download) +/// -> If the services field is empty, just emit and log inside the executed binary "that states a new service instance got created by one of these blueprints" +/// -> If the services field is not empty, for each service in RpcServicesWithBlueprint.services, spawn the gadget binary, using params to set the job type to listen to (in terms of our old language, each spawned service represents a single "RoleType") +async fn handle_init( + tangle_runtime: &TangleClient, + services_client: &TangleServicesClient, + sub_account_id: &AccountId32, + active_gadgets: &mut ActiveGadgets, + gadget_config: &GadgetConfiguration, + blueprint_manager_config: &BlueprintManagerConfig, +) -> Result> { + info!("Beginning initialization of Blueprint Manager"); + + let Some(init_event) = tangle_runtime.next_event().await else { + return Err(Error::InitialBlock); + }; + + let operator_subscribed_blueprints = services_client + .query_operator_blueprints(init_event.hash, sub_account_id.clone()) + .await?; + + info!( + "Received {} initial blueprints this operator is registered to", + operator_subscribed_blueprints.len() + ); + + // Immediately poll, handling the initial state + let poll_result = + event_handler::check_blueprint_events(&init_event, active_gadgets, sub_account_id); + + event_handler::handle_tangle_event( + &init_event, + &operator_subscribed_blueprints, + gadget_config, + blueprint_manager_config, + active_gadgets, + poll_result, + services_client, + ) + .await?; + + Ok(operator_subscribed_blueprints) +} diff --git a/crates/blueprint/manager/src/gadget/mod.rs b/crates/blueprint/manager/src/gadget/mod.rs new file mode 100644 index 0000000..e6cc053 --- /dev/null +++ b/crates/blueprint/manager/src/gadget/mod.rs @@ -0,0 +1,7 @@ +use gadget_std::collections::HashMap; +use gadget_std::sync::atomic::AtomicBool; +use gadget_std::sync::Arc; + +pub type ActiveGadgets = + HashMap, Option>)>>; +pub mod native; diff --git a/crates/blueprint/manager/src/gadget/native.rs b/crates/blueprint/manager/src/gadget/native.rs new file mode 100644 index 0000000..ffff209 --- /dev/null +++ b/crates/blueprint/manager/src/gadget/native.rs @@ -0,0 +1,37 @@ +use crate::sdk::utils::get_formatted_os_string; +use gadget_config::Protocol; +use tangle_subxt::tangle_testnet_runtime::api::runtime_types::tangle_primitives::services::{ + Gadget, GadgetBinary, +}; + +pub struct FilteredBlueprint { + pub blueprint_id: u64, + pub services: Vec, + pub gadget: Gadget, + pub name: String, + pub registration_mode: bool, + pub protocol: Protocol, +} + +pub fn get_gadget_binary(gadget_binaries: &[GadgetBinary]) -> Option<&GadgetBinary> { + let os = get_formatted_os_string().to_lowercase(); + let arch = std::env::consts::ARCH.to_lowercase(); + for binary in gadget_binaries { + let binary_str = format!("{:?}", binary.os).to_lowercase(); + if binary_str.contains(&os) || os.contains(&binary_str) || binary_str == os { + let mut arch_str = format!("{:?}", binary.arch).to_lowercase(); + + if arch_str == "amd" { + arch_str = "x86".to_string() + } else if arch_str == "amd64" { + arch_str = "x86_64".to_string() + } + + if arch_str == arch { + return Some(binary); + } + } + } + + None +} diff --git a/crates/blueprint/manager/src/lib.rs b/crates/blueprint/manager/src/lib.rs new file mode 100644 index 0000000..70e8063 --- /dev/null +++ b/crates/blueprint/manager/src/lib.rs @@ -0,0 +1,15 @@ +pub mod config; +pub mod error; +pub mod executor; +pub mod gadget; +pub mod sdk; +pub mod sources; +pub use executor::run_blueprint_manager; + +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/crates/blueprint/manager/src/main.rs b/crates/blueprint/manager/src/main.rs new file mode 100644 index 0000000..0e06d14 --- /dev/null +++ b/crates/blueprint/manager/src/main.rs @@ -0,0 +1,38 @@ +use blueprint_manager::config::BlueprintManagerConfig; +use blueprint_manager::sdk; +use clap::Parser; +use sdk::entry; + +#[tokio::main] +#[allow(clippy::needless_return)] +async fn main() -> color_eyre::Result<()> { + color_eyre::install()?; + let mut blueprint_manager_config = BlueprintManagerConfig::parse(); + + blueprint_manager_config.data_dir = std::path::absolute(&blueprint_manager_config.data_dir)?; + + entry::setup_blueprint_manager_logger( + blueprint_manager_config.verbose, + blueprint_manager_config.pretty, + "gadget", + )?; + + // TODO: blueprint-manager CLI mode + eprintln!("TODO: blueprint-manager CLI mode"); + return Ok(()); + + // let gadget_config_settings = std::fs::read_to_string(gadget_config)?; + // let gadget_config: GadgetConfig = + // toml::from_str(&gadget_config_settings).map_err(|err| msg_to_error(err.to_string()))?; + // + // // Allow CTRL-C to shutdown this CLI application instance + // let shutdown_signal = async move { + // let _ = tokio::signal::ctrl_c().await; + // }; + // + // let handle = + // run_blueprint_manager(blueprint_manager_config, gadget_config, shutdown_signal).await?; + // handle.await?; + // + // Ok(()) +} diff --git a/crates/blueprint/manager/src/sdk/entry.rs b/crates/blueprint/manager/src/sdk/entry.rs new file mode 100644 index 0000000..44523ba --- /dev/null +++ b/crates/blueprint/manager/src/sdk/entry.rs @@ -0,0 +1,39 @@ +use futures::Future; +use tracing_subscriber::EnvFilter; + +pub trait SendFuture<'a, T>: Send + Future + 'a {} +impl<'a, F: Send + Future + 'a, T> SendFuture<'a, T> for F {} + +/// Sets up the logger for the blueprint manager, based on the verbosity level passed in. +pub fn setup_blueprint_manager_logger( + verbose: u8, + pretty: bool, + filter: &str, +) -> color_eyre::Result<()> { + use tracing::Level; + let log_level = match verbose { + 0 => Level::ERROR, + 1 => Level::WARN, + 2 => Level::INFO, + 3 => Level::DEBUG, + _ => Level::TRACE, + }; + let env_filter = + EnvFilter::from_default_env().add_directive(format!("{filter}={log_level}").parse()?); + let logger = tracing_subscriber::fmt() + .with_target(false) + .with_level(true) + .with_line_number(false) + .without_time() + .with_max_level(log_level) + .with_env_filter(env_filter); + if pretty { + let _ = logger.pretty().try_init(); + } else { + let _ = logger.compact().try_init(); + } + + //let _ = env_logger::try_init(); + + Ok(()) +} diff --git a/crates/blueprint/manager/src/sdk/mod.rs b/crates/blueprint/manager/src/sdk/mod.rs new file mode 100644 index 0000000..555eb18 --- /dev/null +++ b/crates/blueprint/manager/src/sdk/mod.rs @@ -0,0 +1,2 @@ +pub mod entry; +pub mod utils; diff --git a/crates/blueprint/manager/src/sdk/utils.rs b/crates/blueprint/manager/src/sdk/utils.rs new file mode 100644 index 0000000..278b3d0 --- /dev/null +++ b/crates/blueprint/manager/src/sdk/utils.rs @@ -0,0 +1,107 @@ +use crate::error::Result; +use gadget_logging::{info, warn}; +use sha2::Digest; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use tangle_subxt::tangle_testnet_runtime::api::runtime_types::tangle_primitives::services::field::BoundedString; +use tangle_subxt::tangle_testnet_runtime::api::runtime_types::tangle_primitives::services::{ + GadgetBinary, GithubFetcher, +}; + +pub fn bounded_string_to_string(string: BoundedString) -> Result { + let bytes: &Vec = &string.0 .0; + let ret = String::from_utf8(bytes.clone())?; + Ok(ret) +} + +pub fn hash_bytes_to_hex>(input: T) -> String { + let mut hasher = sha2::Sha256::default(); + hasher.update(input); + hex::encode(hasher.finalize()) +} + +pub async fn valid_file_exists(path: &str, expected_hash: &str) -> bool { + // The hash is sha3_256 of the binary + if let Ok(file) = tokio::fs::read(path).await { + // Compute the SHA3-256 + let retrieved_bytes = hash_bytes_to_hex(file); + expected_hash == retrieved_bytes.as_str() + } else { + false + } +} + +pub fn get_formatted_os_string() -> String { + let os = std::env::consts::OS; + + match os { + "macos" => "apple-darwin".to_string(), + "windows" => "pc-windows-msvc".to_string(), + "linux" => "unknown-linux-gnu".to_string(), + _ => os.to_string(), + } +} + +pub fn get_download_url(binary: &GadgetBinary, fetcher: &GithubFetcher) -> String { + let os = get_formatted_os_string(); + let ext = if os == "windows" { ".exe" } else { "" }; + let owner = String::from_utf8(fetcher.owner.0 .0.clone()).expect("Should be a valid owner"); + let repo = String::from_utf8(fetcher.repo.0 .0.clone()).expect("Should be a valid repo"); + let tag = String::from_utf8(fetcher.tag.0 .0.clone()).expect("Should be a valid tag"); + let binary_name = + String::from_utf8(binary.name.0 .0.clone()).expect("Should be a valid binary name"); + let os_name = format!("{:?}", binary.os).to_lowercase(); + let arch_name = format!("{:?}", binary.arch).to_lowercase(); + // https://github.com///releases/download/v/ + format!("https://github.com/{owner}/{repo}/releases/download/v{tag}/{binary_name}-{os_name}-{arch_name}{ext}") +} + +pub fn make_executable>(path: P) -> Result<()> { + #[cfg(target_family = "unix")] + { + use std::os::unix::fs::PermissionsExt; + + let f = std::fs::File::open(path)?; + let mut perms = f.metadata()?.permissions(); + perms.set_mode(perms.mode() | 0o111); + f.set_permissions(perms)?; + } + + Ok(()) +} + +pub fn generate_running_process_status_handle( + process: tokio::process::Child, + service_name: &str, +) -> (Arc, tokio::sync::oneshot::Sender<()>) { + let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); + let status = Arc::new(AtomicBool::new(true)); + let status_clone = status.clone(); + let service_name = service_name.to_string(); + + let task = async move { + info!("Starting process execution for {service_name}"); + let output = process.wait_with_output().await; + warn!("Process for {service_name} exited: {output:?}"); + status_clone.store(false, Ordering::Relaxed); + }; + + let task = async move { + tokio::select! { + _ = stop_rx => {}, + _ = task => {}, + } + }; + + tokio::spawn(task); + (status, stop_tx) +} + +pub fn slice_32_to_sha_hex_string(hash: [u8; 32]) -> String { + use std::fmt::Write; + hash.iter().fold(String::new(), |mut acc, byte| { + write!(&mut acc, "{:02x}", byte).expect("Should be able to write"); + acc + }) +} diff --git a/crates/blueprint/manager/src/sources/github.rs b/crates/blueprint/manager/src/sources/github.rs new file mode 100644 index 0000000..a317a60 --- /dev/null +++ b/crates/blueprint/manager/src/sources/github.rs @@ -0,0 +1,66 @@ +use crate::error::{Error, Result}; +use crate::gadget::native::get_gadget_binary; +use crate::sdk; +use crate::sdk::utils::{get_download_url, hash_bytes_to_hex, valid_file_exists}; +use crate::sources::BinarySourceFetcher; +use async_trait::async_trait; +use gadget_logging::info; +use std::path::PathBuf; +use tangle_subxt::tangle_testnet_runtime::api::runtime_types::tangle_primitives::services::GithubFetcher; +use tokio::io::AsyncWriteExt; + +pub struct GithubBinaryFetcher { + pub fetcher: GithubFetcher, + pub blueprint_id: u64, + pub gadget_name: String, +} + +#[async_trait] +impl BinarySourceFetcher for GithubBinaryFetcher { + async fn get_binary(&self) -> Result { + let relevant_binary = + get_gadget_binary(&self.fetcher.binaries.0).ok_or(Error::NoMatchingBinary)?; + let expected_hash = sdk::utils::slice_32_to_sha_hex_string(relevant_binary.sha256); + let current_dir = std::env::current_dir()?; + let mut binary_download_path = + format!("{}/protocol-{:?}", current_dir.display(), self.fetcher.tag); + + if cfg!(target_family = "windows") { + binary_download_path += ".exe" + } + + info!("Downloading to {binary_download_path}"); + + // Check if the binary exists, if not download it + if !valid_file_exists(&binary_download_path, &expected_hash).await { + return Ok(PathBuf::from(binary_download_path)); + } + + let url = get_download_url(relevant_binary, &self.fetcher); + + let download = reqwest::get(&url).await?.bytes().await?; + let retrieved_hash = hash_bytes_to_hex(&download); + + // Write the binary to disk + let mut file = tokio::fs::File::create(&binary_download_path).await?; + file.write_all(&download).await?; + file.flush().await?; + + if retrieved_hash.trim() != expected_hash.trim() { + return Err(Error::HashMismatch { + expected: expected_hash, + actual: retrieved_hash, + }); + } + + Ok(PathBuf::from(binary_download_path)) + } + + fn blueprint_id(&self) -> u64 { + self.blueprint_id + } + + fn name(&self) -> String { + self.gadget_name.clone() + } +} diff --git a/crates/blueprint/manager/src/sources/mod.rs b/crates/blueprint/manager/src/sources/mod.rs new file mode 100644 index 0000000..c223c7c --- /dev/null +++ b/crates/blueprint/manager/src/sources/mod.rs @@ -0,0 +1,99 @@ +use crate::config::BlueprintManagerConfig; +use crate::error::Result; +use crate::gadget::native::FilteredBlueprint; +use async_trait::async_trait; +use gadget_config::GadgetConfiguration; +use std::path::PathBuf; + +pub mod github; +pub mod testing; + +#[async_trait] +#[auto_impl::auto_impl(Box)] +pub trait BinarySourceFetcher: Send + Sync { + async fn get_binary(&self) -> Result; + fn blueprint_id(&self) -> u64; + fn name(&self) -> String; +} + +pub fn process_arguments_and_env( + gadget_config: &GadgetConfiguration, + manager_opts: &BlueprintManagerConfig, + blueprint_id: u64, + service_id: u64, + blueprint: &FilteredBlueprint, + sub_service_str: &str, +) -> (Vec, Vec<(String, String)>) { + let mut arguments = vec![]; + arguments.push("run".to_string()); + + if manager_opts.test_mode { + arguments.push("--test-mode".to_string()); + } + + if manager_opts.pretty { + arguments.push("--pretty".to_string()); + } + + for bootnode in &gadget_config.bootnodes { + arguments.push(format!("--bootnodes={}", bootnode)); + } + + arguments.extend([ + format!("--http-rpc-url={}", gadget_config.http_rpc_endpoint), + format!("--ws-rpc-url={}", gadget_config.ws_rpc_endpoint), + format!("--keystore-uri={}", gadget_config.keystore_uri), + format!("--protocol={}", blueprint.protocol), + format!( + "--log-id=Blueprint-{blueprint_id}-Service-{service_id}{}", + manager_opts + .instance_id + .clone() + .map_or(String::new(), |id| format!("-{}", id)) + ), + ]); + + // TODO: Add support for keystore password + // if let Some(keystore_password) = &gadget_config.keystore_password { + // arguments.push(format!("--keystore-password={}", keystore_password)); + // } + + // Uses occurrences of clap short -v + if manager_opts.verbose > 0 { + arguments.push(format!("-{}", "v".repeat(manager_opts.verbose as usize))); + } + + // Add required env vars for all child processes/gadgets + let mut env_vars = vec![ + ( + "HTTP_RPC_URL".to_string(), + gadget_config.http_rpc_endpoint.to_string(), + ), + ( + "WS_RPC_URL".to_string(), + gadget_config.ws_rpc_endpoint.to_string(), + ), + ( + "KEYSTORE_URI".to_string(), + manager_opts.keystore_uri.clone(), + ), + ("BLUEPRINT_ID".to_string(), format!("{}", blueprint_id)), + ("SERVICE_ID".to_string(), format!("{}", service_id)), + ]; + + let base_data_dir = &manager_opts.data_dir; + let data_dir = base_data_dir.join(format!("blueprint-{blueprint_id}-{sub_service_str}")); + env_vars.push(( + "DATA_DIR".to_string(), + data_dir.to_string_lossy().into_owned(), + )); + + // Ensure our child process inherits the current processes' environment vars + env_vars.extend(std::env::vars()); + + if blueprint.registration_mode { + env_vars.push(("REGISTRATION_MODE_ON".to_string(), "true".to_string())); + } + + (arguments, env_vars) +} diff --git a/crates/blueprint/manager/src/sources/testing.rs b/crates/blueprint/manager/src/sources/testing.rs new file mode 100644 index 0000000..80bb3b8 --- /dev/null +++ b/crates/blueprint/manager/src/sources/testing.rs @@ -0,0 +1,88 @@ +use crate::error::{Error, Result}; +use crate::sources::BinarySourceFetcher; +use async_trait::async_trait; +use gadget_logging::trace; +use std::path::PathBuf; +use tangle_subxt::tangle_testnet_runtime::api::runtime_types::tangle_primitives::services::TestFetcher; + +pub struct TestSourceFetcher { + pub fetcher: TestFetcher, + pub blueprint_id: u64, + pub gadget_name: String, +} + +#[async_trait] +impl BinarySourceFetcher for TestSourceFetcher { + async fn get_binary(&self) -> Result { + // Step 1: Build the binary. It will be stored in the root directory/bin/ + let TestFetcher { + cargo_package, + base_path, + .. + } = &self.fetcher; + let cargo_bin = String::from_utf8(cargo_package.0 .0.clone()) + .map_err(|err| Error::Other(format!("Failed to parse `cargo_bin`: {:?}", err)))?; + let base_path_str = String::from_utf8(base_path.0 .0.clone()) + .map_err(|err| Error::Other(format!("Failed to parse `base_path`: {:?}", err)))?; + let git_repo_root = get_git_repo_root_path().await?; + + let profile = if cfg!(debug_assertions) { + "debug" + } else { + "release" + }; + let base_path = std::path::absolute(git_repo_root.join(&base_path_str))?; + + let target_dir = match std::env::var("CARGO_TARGET_DIR") { + Ok(target) => PathBuf::from(target), + Err(_) => git_repo_root.join(&base_path).join("target"), + }; + + let binary_path = target_dir.join(profile).join(&cargo_bin); + let binary_path = std::path::absolute(&binary_path)?; + + trace!("Base Path: {}", base_path.display()); + trace!("Binary Path: {}", binary_path.display()); + + // Run cargo build on the cargo_bin and ensure it build to the binary_path + let mut command = tokio::process::Command::new("cargo"); + command + .arg("build") + .arg(format!("--target-dir={}", target_dir.display())) + .arg("--bin") + .arg(&cargo_bin); + + if !cfg!(debug_assertions) { + command.arg("--release"); + } + + let output = command.current_dir(&base_path).output().await?; + if !output.status.success() { + return Err(Error::BuildBinary(output)); + } + + Ok(binary_path) + } + + fn blueprint_id(&self) -> u64 { + self.blueprint_id + } + + fn name(&self) -> String { + self.gadget_name.clone() + } +} +async fn get_git_repo_root_path() -> Result { + // Run a process to determine the root directory for this repo + let output = tokio::process::Command::new("git") + .arg("rev-parse") + .arg("--show-toplevel") + .output() + .await?; + + if !output.status.success() { + return Err(Error::FetchGitRoot(output)); + } + + Ok(PathBuf::from(String::from_utf8(output.stdout)?.trim())) +} diff --git a/crates/clients/Cargo.toml b/crates/clients/Cargo.toml index bee79db..21127e5 100644 --- a/crates/clients/Cargo.toml +++ b/crates/clients/Cargo.toml @@ -10,6 +10,8 @@ gadget-client-tangle = { workspace = true, optional = true } gadget-client-networking = { workspace = true, optional = true } gadget-client-core = { workspace = true } +thiserror.workspace = true + [features] default = ["std"] std = [ @@ -17,6 +19,7 @@ std = [ "gadget-client-evm?/std", "gadget-client-networking?/std", "gadget-client-tangle?/std", + "thiserror/std" ] web = ["gadget-client-tangle?/web"] diff --git a/crates/clients/core/src/lib.rs b/crates/clients/core/src/lib.rs index eb9e23c..ba6470c 100644 --- a/crates/clients/core/src/lib.rs +++ b/crates/clients/core/src/lib.rs @@ -13,19 +13,24 @@ pub trait GadgetServicesClient: Send + Sync + 'static { type PublicAccountIdentity: Send + Sync + 'static; /// A generalized ID that distinguishes the current blueprint from others type Id: Send + Sync + 'static; + type Error: core::error::Error + From + Send + Sync + 'static; + /// Returns the set of operators for the current job async fn get_operators( &self, - ) -> Result, Error>; + ) -> Result< + OperatorSet, + Self::Error, + >; /// Returns the ID of the operator - async fn operator_id(&self) -> Result; + async fn operator_id(&self) -> Result; /// Returns the unique ID for this blueprint - async fn blueprint_id(&self) -> Result; + async fn blueprint_id(&self) -> Result; /// Returns an operator set with the index of the current operator within that set async fn get_operators_and_operator_id( &self, - ) -> Result<(OperatorSet, usize), Error> { + ) -> Result<(OperatorSet, usize), Self::Error> { let operators = self .get_operators() .await @@ -51,11 +56,12 @@ pub trait GadgetServicesClient: Send + Sync + 'static { } /// Returns the index of the current operator in the operator set - async fn get_operator_index(&self) -> Result { - self.get_operators_and_operator_id() + async fn get_operator_index(&self) -> Result { + let (_, index) = self + .get_operators_and_operator_id() .await - .map_err(|err| Error::GetOperatorIndex(err.to_string())) - .map(|(_, index)| index) + .map_err(|err| Error::GetOperatorIndex(err.to_string()))?; + Ok(index) } } diff --git a/crates/clients/eigenlayer/src/error.rs b/crates/clients/eigenlayer/src/error.rs index 6d64c15..04f8fe6 100644 --- a/crates/clients/eigenlayer/src/error.rs +++ b/crates/clients/eigenlayer/src/error.rs @@ -2,7 +2,7 @@ use gadget_std::string::ParseError; use thiserror::Error; #[derive(Debug, Error)] -pub enum EigenlayerClientError { +pub enum Error { #[error("IO error: {0}")] Io(#[from] gadget_std::io::Error), #[error("Parse error {0}")] @@ -25,10 +25,10 @@ pub enum EigenlayerClientError { OtherStatic(&'static str), } -impl From<&'static str> for EigenlayerClientError { +impl From<&'static str> for Error { fn from(e: &'static str) -> Self { - EigenlayerClientError::OtherStatic(e) + Error::OtherStatic(e) } } -pub type Result = gadget_std::result::Result; +pub type Result = gadget_std::result::Result; diff --git a/crates/clients/evm/src/error.rs b/crates/clients/evm/src/error.rs index 688dca5..df41f37 100644 --- a/crates/clients/evm/src/error.rs +++ b/crates/clients/evm/src/error.rs @@ -2,7 +2,7 @@ use gadget_std::string::String; use thiserror::Error; #[derive(Debug, Error)] -pub enum EvmError { +pub enum Error { #[error("Provider error: {0}")] Provider(String), #[error("Invalid address: {0}")] @@ -15,4 +15,4 @@ pub enum EvmError { Abi(String), } -pub type Result = gadget_std::result::Result; +pub type Result = gadget_std::result::Result; diff --git a/crates/clients/networking/src/error.rs b/crates/clients/networking/src/error.rs index 5f4c51d..710fb67 100644 --- a/crates/clients/networking/src/error.rs +++ b/crates/clients/networking/src/error.rs @@ -2,7 +2,7 @@ use gadget_std::string::String; use thiserror::Error; #[derive(Debug, Error)] -pub enum NetworkError { +pub enum Error { #[error("P2P error: {0}")] P2p(String), #[error("Transport error: {0}")] @@ -13,4 +13,4 @@ pub enum NetworkError { Configuration(String), } -pub type Result = gadget_std::result::Result; +pub type Result = gadget_std::result::Result; diff --git a/crates/clients/networking/src/p2p.rs b/crates/clients/networking/src/p2p.rs index 9ab9330..fc5c9bd 100644 --- a/crates/clients/networking/src/p2p.rs +++ b/crates/clients/networking/src/p2p.rs @@ -1,4 +1,4 @@ -use crate::error::{NetworkError, Result}; +use crate::error::{Error, Result}; use gadget_config::GadgetConfiguration; use gadget_crypto::k256_crypto::{K256SigningKey, K256VerifyingKey}; use gadget_networking::gossip::GossipHandle; @@ -35,7 +35,7 @@ impl P2PClient { pub fn libp2p_identity(&self, ed25519_seed: Vec) -> Result { let mut seed_bytes = ed25519_seed; let keypair = libp2p::identity::Keypair::ed25519_from_bytes(&mut seed_bytes) - .map_err(|err| NetworkError::Configuration(err.to_string()))?; + .map_err(|err| Error::Configuration(err.to_string()))?; Ok(keypair) } @@ -68,9 +68,7 @@ impl P2PClient { Ok(handle) => Ok(handle), Err(err) => { gadget_logging::error!("Failed to start network: {}", err.to_string()); - Err(NetworkError::Protocol(format!( - "Failed to start network: {err}" - ))) + Err(Error::Protocol(format!("Failed to start network: {err}"))) } } } diff --git a/crates/clients/src/error.rs b/crates/clients/src/error.rs new file mode 100644 index 0000000..bbe7735 --- /dev/null +++ b/crates/clients/src/error.rs @@ -0,0 +1,17 @@ +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + Core(#[from] gadget_client_core::Error), + #[error(transparent)] + #[cfg(feature = "eigenlayer")] + Eigenlayer(#[from] gadget_client_eigenlayer::error::Error), + #[error(transparent)] + #[cfg(feature = "evm")] + Evm(#[from] gadget_client_evm::error::Error), + #[error(transparent)] + #[cfg(feature = "networking")] + Networking(#[from] gadget_client_networking::error::Error), + #[error(transparent)] + #[cfg(feature = "tangle")] + Tangle(#[from] gadget_client_tangle::error::Error), +} diff --git a/crates/clients/src/lib.rs b/crates/clients/src/lib.rs index d6e6c30..e82d0b2 100644 --- a/crates/clients/src/lib.rs +++ b/crates/clients/src/lib.rs @@ -1,5 +1,7 @@ #![cfg_attr(not(feature = "std"), no_std)] +pub mod error; + #[cfg(feature = "eigenlayer")] pub use gadget_client_eigenlayer as eigenlayer; diff --git a/crates/clients/tangle/src/client.rs b/crates/clients/tangle/src/client.rs index f7c8ac7..2c7a76c 100644 --- a/crates/clients/tangle/src/client.rs +++ b/crates/clients/tangle/src/client.rs @@ -1,5 +1,5 @@ use sp_core::ecdsa; -use crate::error::Result; +use crate::error::{Result, Error}; use crate::EventsClient; use gadget_std::sync::Arc; use gadget_std::time::Duration; @@ -10,7 +10,7 @@ use subxt::utils::AccountId32; use subxt::{self, PolkadotConfig}; use tangle_subxt::tangle_testnet_runtime::api; use tangle_subxt::tangle_testnet_runtime::api::runtime_types::pallet_multi_asset_delegation::types::operator::OperatorMetadata; -use gadget_client_core::{Error, GadgetServicesClient, OperatorSet}; +use gadget_client_core::{GadgetServicesClient, OperatorSet}; use gadget_config::GadgetConfiguration; use gadget_crypto_sp_core::{SpEcdsa, SpSr25519}; use gadget_keystore::{Keystore, KeystoreConfig}; @@ -56,18 +56,13 @@ impl TangleClient { keystore_config.fs_root(config.keystore_uri.replace("file://", "")) }; - let keystore = Arc::new(Keystore::new(keystore_config).map_err(|err| Error::msg(err))?); + let keystore = Arc::new(Keystore::new(keystore_config)?); let rpc_url = config.ws_rpc_endpoint.as_str(); - let client = TangleServicesClient::new( - subxt::OnlineClient::from_url(rpc_url) - .await - .map_err(|err| Error::msg(err))?, - ); + let client = TangleServicesClient::new(subxt::OnlineClient::from_url(rpc_url).await?); let account_id = keystore - .get_public_key_local::(KEY_ID) - .map_err(|err| Error::msg(err))? + .get_public_key_local::(KEY_ID)? .0 .0 .into(); @@ -137,13 +132,11 @@ impl TangleClient { .rpc_client .storage() .at_latest() - .await - .map_err(|err| Error::msg(err))?; + .await?; let metadata_storage_key = api::storage().multi_asset_delegation().operators(operator); - storage - .fetch(&metadata_storage_key) - .await - .map_err(|err| Error::msg(err)) + + let ret = storage.fetch(&metadata_storage_key).await?; + Ok(ret) } /// Retrieves the current party index and operator mapping @@ -162,10 +155,7 @@ impl TangleClient { Error, > { let parties = self.get_operators().await?; - let my_id = self - .keystore - .get_public_key_local::(KEY_ID) - .map_err(|err| Error::msg(err))?; + let my_id = self.keystore.get_public_key_local::(KEY_ID)?; gadget_logging::trace!( "Looking for {my_id:?} in parties: {:?}", @@ -175,7 +165,7 @@ impl TangleClient { let index_of_my_id = parties .iter() .position(|(_id, key)| key == &my_id.0) - .ok_or_else(|| Error::msg("Party not found in operator list"))?; + .ok_or(Error::PartyNotFound)?; Ok((index_of_my_id, parties)) } @@ -241,6 +231,7 @@ impl GadgetServicesClient for TangleClient { type PublicApplicationIdentity = ecdsa::Public; type PublicAccountIdentity = AccountId32; type Id = BlueprintId; + type Error = Error; /// Retrieves the ECDSA keys for all current service operators /// @@ -253,7 +244,7 @@ impl GadgetServicesClient for TangleClient { &self, ) -> std::result::Result< OperatorSet, - Error, + Self::Error, > { let client = &self.services_client; let current_blueprint = self.blueprint_id().await?; @@ -261,7 +252,7 @@ impl GadgetServicesClient for TangleClient { .config .protocol_settings .tangle() - .map_err(|err| Error::msg(err))? + .map_err(|_| Error::NotTangle)? .service_id .ok_or_else(|| Error::Other("No service ID injected into config".into()))?; let now = self @@ -271,14 +262,8 @@ impl GadgetServicesClient for TangleClient { let current_service_op = self .services_client .current_service_operators(now, service_id) - .await - .map_err(|err| Error::msg(err))?; - let storage = client - .rpc_client - .storage() - .at_latest() - .await - .map_err(|err| Error::msg(err))?; + .await?; + let storage = client.rpc_client.storage().at_latest().await?; let mut map = std::collections::BTreeMap::new(); for (operator, _) in current_service_op { @@ -287,7 +272,7 @@ impl GadgetServicesClient for TangleClient { .operators(current_blueprint, &operator); let maybe_pref = storage.fetch(&addr).await.map_err(|err| { - Error::msg(format!( + Error::Other(format!( "Failed to fetch operator storage for {operator}: {err}" )) })?; @@ -295,35 +280,29 @@ impl GadgetServicesClient for TangleClient { if let Some(pref) = maybe_pref { map.insert(operator, ecdsa::Public(pref.key)); } else { - return Err(Error::msg(format!( - "Missing ECDSA key for operator {operator}" - ))); + return Err(Error::MissingEcdsa(operator)); } } Ok(map) } - async fn operator_id(&self) -> std::result::Result { - Ok(self - .keystore - .get_public_key_local::(KEY_ID) - .map_err(|err| Error::msg(err))? - .0) + async fn operator_id( + &self, + ) -> std::result::Result { + Ok(self.keystore.get_public_key_local::(KEY_ID)?.0) } /// Retrieves the current blueprint ID from the configuration /// /// # Errors /// Returns an error if the blueprint ID is not found in the configuration - async fn blueprint_id(&self) -> std::result::Result { - let id = self + async fn blueprint_id(&self) -> std::result::Result { + let c = self .config .protocol_settings .tangle() - .map(|c| c.blueprint_id) - .map_err(|err| format!("Blueprint ID not found in configuration: {err}")) - .map_err(|err| Error::msg(err))?; - Ok(id) + .map_err(|_| Error::NotTangle)?; + Ok(c.blueprint_id) } } diff --git a/crates/clients/tangle/src/error.rs b/crates/clients/tangle/src/error.rs index 18fbe17..ba23f19 100644 --- a/crates/clients/tangle/src/error.rs +++ b/crates/clients/tangle/src/error.rs @@ -1,20 +1,34 @@ use gadget_std::io; use gadget_std::string::String; +use subxt_core::utils::AccountId32; use thiserror::Error; #[derive(Debug, Error)] -pub enum TangleClientError { - #[error("IO error: {0}")] - Io(#[from] io::Error), - #[error("Subxt error: {0}")] - Subxt(#[from] subxt::Error), +pub enum Error { #[error("Tangle error: {0}")] Tangle(TangleDispatchError), + #[error("Not a Tangle instance")] + NotTangle, + + #[error("Missing ECDSA key for operator: {0}")] + MissingEcdsa(AccountId32), + #[error("Party not found in operator list")] + PartyNotFound, + #[error("{0}")] Other(String), + + #[error(transparent)] + Keystore(#[from] gadget_keystore::Error), + #[error(transparent)] + Core(#[from] gadget_client_core::Error), + #[error("IO error: {0}")] + Io(#[from] io::Error), + #[error("Subxt error: {0}")] + Subxt(#[from] subxt::Error), } -pub type Result = gadget_std::result::Result; +pub type Result = gadget_std::result::Result; #[derive(Debug)] pub struct TangleDispatchError( @@ -31,9 +45,9 @@ impl From for TangleClientError { +impl From for Error { fn from(error: TangleDispatchError) -> Self { - TangleClientError::Tangle(error) + Error::Tangle(error) } } diff --git a/crates/clients/tangle/src/services.rs b/crates/clients/tangle/src/services.rs index 1878ab9..725acc2 100644 --- a/crates/clients/tangle/src/services.rs +++ b/crates/clients/tangle/src/services.rs @@ -1,4 +1,4 @@ -use crate::error::TangleClientError; +use crate::error::Error; use crate::error::{Result, TangleDispatchError}; use gadget_std::string::ToString; use gadget_std::vec::Vec; @@ -88,7 +88,7 @@ where let ret = self.rpc_client.storage().at(at).fetch(&call).await?; match ret { Some(blueprints) => Ok(blueprints.1), - None => Err(TangleClientError::Other("Blueprint not found".to_string())), + None => Err(Error::Other("Blueprint not found".to_string())), } } @@ -103,7 +103,7 @@ where let ret = self.rpc_client.storage().at(at).fetch(&call).await?; match ret { Some(blueprints) => Ok(blueprints.0), - None => Err(TangleClientError::Other("Blueprint not found".to_string())), + None => Err(Error::Other("Blueprint not found".to_string())), } } diff --git a/crates/crypto/tangle-pair-signer/src/lib.rs b/crates/crypto/tangle-pair-signer/src/lib.rs index 84c198e..e62d4af 100644 --- a/crates/crypto/tangle-pair-signer/src/lib.rs +++ b/crates/crypto/tangle-pair-signer/src/lib.rs @@ -1,6 +1,125 @@ #![cfg_attr(not(feature = "std"), no_std)] pub mod error; -pub mod tangle_pair_signer; -pub use tangle_pair_signer::*; +use gadget_std::vec::Vec; +pub use sp_core; +use sp_core::crypto::DeriveError; +use sp_core::crypto::SecretStringError; +use sp_core::DeriveJunction; +use subxt::PolkadotConfig; +use subxt_core::{ + tx::signer::{PairSigner, Signer}, + utils::{AccountId32, MultiAddress, MultiSignature}, +}; + +#[derive(Clone, Debug)] +pub struct TanglePairSigner { + pub(crate) pair: subxt::tx::PairSigner, +} + +impl sp_core::crypto::CryptoType for TanglePairSigner { + type Pair = Pair; +} + +impl TanglePairSigner +where + ::Signature: Into, + subxt::ext::sp_runtime::MultiSigner: From<::Public>, +{ + pub fn new(pair: Pair) -> Self { + TanglePairSigner { + pair: PairSigner::new(pair), + } + } + + pub fn into_inner(self) -> PairSigner { + self.pair + } + + pub fn signer(&self) -> &Pair { + self.pair.signer() + } +} + +impl Signer for TanglePairSigner +where + Pair: sp_core::Pair, + Pair::Signature: Into, +{ + fn account_id(&self) -> AccountId32 { + self.pair.account_id() + } + + fn address(&self) -> MultiAddress { + self.pair.address() + } + + fn sign(&self, signer_payload: &[u8]) -> MultiSignature { + self.pair.sign(signer_payload) + } +} + +impl sp_core::Pair for TanglePairSigner +where + ::Signature: Into, + subxt::ext::sp_runtime::MultiSigner: From<::Public>, +{ + type Public = Pair::Public; + type Seed = Pair::Seed; + type Signature = Pair::Signature; + + fn derive>( + &self, + path: Iter, + seed: Option, + ) -> Result<(Self, Option), DeriveError> { + Pair::derive(self.pair.signer(), path, seed).map(|(pair, seed)| { + ( + TanglePairSigner { + pair: PairSigner::new(pair), + }, + seed, + ) + }) + } + + fn from_seed_slice(seed: &[u8]) -> Result { + Pair::from_seed_slice(seed).map(|pair| TanglePairSigner { + pair: PairSigner::new(pair), + }) + } + + fn sign(&self, message: &[u8]) -> Self::Signature { + Pair::sign(self.pair.signer(), message) + } + + fn verify>(sig: &Self::Signature, message: M, pubkey: &Self::Public) -> bool { + Pair::verify(sig, message, pubkey) + } + + fn public(&self) -> Self::Public { + Pair::public(self.pair.signer()) + } + + fn to_raw_vec(&self) -> Vec { + Pair::to_raw_vec(self.pair.signer()) + } +} + +#[cfg(feature = "evm")] +impl TanglePairSigner { + /// Returns the alloy-compatible key for the ECDSA key pair. + pub fn alloy_key( + &self, + ) -> crate::error::Result> { + let k256_ecdsa_secret_key = self.pair.signer().seed(); + let res = alloy_signer_local::LocalSigner::from_slice(&k256_ecdsa_secret_key)?; + Ok(res) + } + + /// Returns the Alloy Address for the ECDSA key pair. + pub fn alloy_address(&self) -> crate::error::Result { + Ok(self.alloy_key()?.address()) + } +} diff --git a/crates/crypto/tangle-pair-signer/src/tangle_pair_signer.rs b/crates/crypto/tangle-pair-signer/src/tangle_pair_signer.rs deleted file mode 100644 index 07947ee..0000000 --- a/crates/crypto/tangle-pair-signer/src/tangle_pair_signer.rs +++ /dev/null @@ -1,121 +0,0 @@ -use gadget_std::vec::Vec; -pub use sp_core; -use sp_core::crypto::DeriveError; -use sp_core::crypto::SecretStringError; -use sp_core::DeriveJunction; -use subxt::PolkadotConfig; -use subxt_core::{ - tx::signer::{PairSigner, Signer}, - utils::{AccountId32, MultiAddress, MultiSignature}, -}; - -#[derive(Clone, Debug)] -pub struct TanglePairSigner { - pub pair: subxt::tx::PairSigner, -} - -impl sp_core::crypto::CryptoType for TanglePairSigner { - type Pair = Pair; -} - -impl TanglePairSigner -where - ::Signature: Into, - subxt::ext::sp_runtime::MultiSigner: From<::Public>, -{ - pub fn new(pair: Pair) -> Self { - TanglePairSigner { - pair: PairSigner::new(pair), - } - } - - pub fn into_inner(self) -> PairSigner { - self.pair - } - - pub fn signer(&self) -> &Pair { - self.pair.signer() - } -} - -impl Signer for TanglePairSigner -where - Pair: sp_core::Pair, - Pair::Signature: Into, -{ - fn account_id(&self) -> AccountId32 { - self.pair.account_id() - } - - fn address(&self) -> MultiAddress { - self.pair.address() - } - - fn sign(&self, signer_payload: &[u8]) -> MultiSignature { - self.pair.sign(signer_payload) - } -} - -impl sp_core::Pair for TanglePairSigner -where - ::Signature: Into, - subxt::ext::sp_runtime::MultiSigner: From<::Public>, -{ - type Public = Pair::Public; - type Seed = Pair::Seed; - type Signature = Pair::Signature; - - fn derive>( - &self, - path: Iter, - seed: Option, - ) -> Result<(Self, Option), DeriveError> { - Pair::derive(self.pair.signer(), path, seed).map(|(pair, seed)| { - ( - TanglePairSigner { - pair: PairSigner::new(pair), - }, - seed, - ) - }) - } - - fn from_seed_slice(seed: &[u8]) -> Result { - Pair::from_seed_slice(seed).map(|pair| TanglePairSigner { - pair: PairSigner::new(pair), - }) - } - - fn sign(&self, message: &[u8]) -> Self::Signature { - Pair::sign(self.pair.signer(), message) - } - - fn verify>(sig: &Self::Signature, message: M, pubkey: &Self::Public) -> bool { - Pair::verify(sig, message, pubkey) - } - - fn public(&self) -> Self::Public { - Pair::public(self.pair.signer()) - } - - fn to_raw_vec(&self) -> Vec { - Pair::to_raw_vec(self.pair.signer()) - } -} - -#[cfg(feature = "evm")] -impl TanglePairSigner { - /// Returns the alloy-compatible key for the ECDSA key pair. - pub fn alloy_key( - &self, - ) -> crate::error::Result> { - let k256_ecdsa_secret_key = self.pair.signer().seed(); - let res = alloy_signer_local::LocalSigner::from_slice(&k256_ecdsa_secret_key)?; - Ok(res) - } - - /// Returns the Alloy Address for the ECDSA key pair. - pub fn alloy_address(&self) -> crate::error::Result { - Ok(self.alloy_key()?.address()) - } -} diff --git a/crates/event-listeners/tangle/Cargo.toml b/crates/event-listeners/tangle/Cargo.toml index e1594cb..239bec9 100644 --- a/crates/event-listeners/tangle/Cargo.toml +++ b/crates/event-listeners/tangle/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] gadget-blueprint-serde = { workspace = true } gadget-clients = { workspace = true, features = ["tangle"] } -gadget-contexts = { workspace = true, features = ["keystore"] } +gadget-contexts = { workspace = true, features = ["keystore", "tangle"] } gadget-crypto-tangle-pair-signer = { workspace = true } gadget-event-listeners-core = { workspace = true } gadget-keystore = { workspace = true } diff --git a/crates/event-listeners/tangle/src/events.rs b/crates/event-listeners/tangle/src/events.rs index 22a9170..abf0d9f 100644 --- a/crates/event-listeners/tangle/src/events.rs +++ b/crates/event-listeners/tangle/src/events.rs @@ -1,7 +1,7 @@ use crate::error::{Result, TangleEventListenerError}; use async_trait::async_trait; use gadget_clients::tangle::client::{OnlineClient, TangleConfig}; -use gadget_crypto_tangle_pair_signer::tangle_pair_signer::TanglePairSigner; +use gadget_crypto_tangle_pair_signer::TanglePairSigner; use gadget_event_listeners_core::marker::IsTangle; use gadget_event_listeners_core::EventListener; use gadget_std::collections::VecDeque; diff --git a/crates/keystore/Cargo.toml b/crates/keystore/Cargo.toml index 737d734..f2166aa 100644 --- a/crates/keystore/Cargo.toml +++ b/crates/keystore/Cargo.toml @@ -121,6 +121,7 @@ tangle = [ "ecdsa", "sr25519-schnorrkel", "zebra", + "gadget-crypto/tangle-pair-signer", ] tangle-bls = [ diff --git a/crates/keystore/src/keystore/backends/tangle.rs b/crates/keystore/src/keystore/backends/tangle.rs index 1cf017e..e74ba4c 100644 --- a/crates/keystore/src/keystore/backends/tangle.rs +++ b/crates/keystore/src/keystore/backends/tangle.rs @@ -3,15 +3,10 @@ use crate::keystore::Keystore; use gadget_crypto::sp_core_crypto::{ SpEcdsaPair, SpEcdsaPublic, SpEd25519Pair, SpEd25519Public, SpSr25519Pair, SpSr25519Public, }; +use gadget_crypto::tangle_pair_signer::TanglePairSigner; use gadget_crypto::KeyTypeId; use sp_core::Pair; use sp_core::{ecdsa, ed25519, sr25519}; -use subxt::tx::PairSigner; -use subxt::PolkadotConfig; - -pub struct TanglePairSigner { - pub pair: subxt::tx::PairSigner, -} #[async_trait::async_trait] pub trait TangleBackend: Send + Sync { @@ -56,9 +51,9 @@ pub trait TangleBackend: Send + Sync { let pair = pair.into(); let seed = pair.as_ref().secret.to_bytes(); let _ = self.sr25519_generate_new(Some(&seed))?; - Ok(TanglePairSigner { - pair: PairSigner::new(sr25519::Pair::from_seed_slice(&seed)?), - }) + Ok(TanglePairSigner::new(sr25519::Pair::from_seed_slice( + &seed, + )?)) } fn create_ed25519_from_pair>( @@ -68,9 +63,9 @@ pub trait TangleBackend: Send + Sync { let pair = pair.into(); let seed = pair.seed(); let _ = self.ed25519_generate_new(Some(&seed))?; - Ok(TanglePairSigner { - pair: PairSigner::new(ed25519::Pair::from_seed_slice(&seed)?), - }) + Ok(TanglePairSigner::new(ed25519::Pair::from_seed_slice( + &seed, + )?)) } fn create_ecdsa_from_pair>( @@ -80,9 +75,7 @@ pub trait TangleBackend: Send + Sync { let pair = pair.into(); let seed = pair.seed(); let _ = self.ecdsa_generate_new(Some(&seed))?; - Ok(TanglePairSigner { - pair: PairSigner::new(ecdsa::Pair::from_seed_slice(&seed)?), - }) + Ok(TanglePairSigner::new(ecdsa::Pair::from_seed_slice(&seed)?)) } } diff --git a/crates/keystore/src/keystore/config.rs b/crates/keystore/src/keystore/config.rs index 2b79cc5..660ea56 100644 --- a/crates/keystore/src/keystore/config.rs +++ b/crates/keystore/src/keystore/config.rs @@ -30,7 +30,7 @@ /// [`InMemoryStorage`]: crate::storage::InMemoryStorage /// [`Keystore`]: crate::Keystore /// [`Keystore::new()`]: crate::Keystore::new -#[derive(Default)] +#[derive(Default, Debug)] pub struct KeystoreConfig { pub(crate) in_memory: bool, #[cfg(feature = "std")] diff --git a/crates/macros/blueprint-proc-macro/src/report.rs b/crates/macros/blueprint-proc-macro/src/report.rs index 9d4b1e6..1a32a94 100644 --- a/crates/macros/blueprint-proc-macro/src/report.rs +++ b/crates/macros/blueprint-proc-macro/src/report.rs @@ -403,7 +403,7 @@ fn generate_qos_report_event_handler( #[automatically_derived] #[gadget_sdk::async_trait::async_trait] - impl gadget_sdk::event_utils::substrate::EventHandler for #struct_name { + impl gadget_sdk::event_utils::substrate::EventHandler for #struct_name { async fn handle(&self, event: &#event_type) -> Result>, gadget_sdk::event_utils::Error> { use std::time::Duration; use gadget_sdk::slashing::reports::{QoSReporter, DefaultQoSReporter}; diff --git a/crates/macros/core/src/lib.rs b/crates/macros/core/src/lib.rs index 703287b..c72c579 100644 --- a/crates/macros/core/src/lib.rs +++ b/crates/macros/core/src/lib.rs @@ -55,7 +55,6 @@ pub enum FieldType { } impl FieldType { - #[must_use] /// Returns the Rust type representation of this field type as a string. /// /// This method converts the `FieldType` enum variant into its corresponding Rust type string. diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 0ae1bd9..16c9114 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.83.0" +channel = "nightly-2024-10-13" components = ["rustfmt", "clippy", "rust-src"] targets = ["wasm32-unknown-unknown"] profile = "minimal"