diff --git a/Cargo.lock b/Cargo.lock index b84c8bf6b..2a536c524 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3623,6 +3623,60 @@ dependencies = [ [[package]] name = "icp" version = "1.5.0" +dependencies = [ + "async-trait", + "bigdecimal", + "camino", + "camino-tempfile", + "candid", + "candid_parser", + "clap", + "dunce", + "flate2", + "futures", + "glob", + "handlebars", + "hex", + "httptest", + "ic-agent", + "ic-ledger-types", + "ic-management-canister-types 0.9.0", + "ic-utils", + "icp-canister-interfaces", + "icp-events", + "icp-sync-plugin", + "icrc-ledger-types", + "indexmap", + "indoc", + "itertools 0.14.0", + "jsonschema", + "num-bigint 0.4.6", + "num-integer", + "num-traits", + "pathdiff", + "rand 0.10.1", + "schemars", + "semver", + "serde", + "serde_cbor", + "serde_json", + "serde_yaml", + "sha2 0.11.0", + "shellwords", + "snafu", + "strum 0.28.0", + "tar", + "time", + "tokio", + "tracing", + "url", + "wasmparser 0.255.0", + "winreg", +] + +[[package]] +name = "icp-app" +version = "1.5.0" dependencies = [ "async-dropper", "async-trait", @@ -3634,7 +3688,6 @@ dependencies = [ "camino", "camino-tempfile", "candid", - "candid_parser", "clap", "crypto-bigint", "directories", @@ -3643,8 +3696,6 @@ dependencies = [ "elliptic-curve", "flate2", "futures", - "glob", - "handlebars", "hex", "hmac 0.13.0", "httptest", @@ -3655,37 +3706,31 @@ dependencies = [ "ic-ledger-types", "ic-management-canister-types 0.9.0", "ic-utils", + "icp", "icp-canister-interfaces", "icp-events", - "icp-sync-plugin", "icrc-ledger-types", "indexmap", "indoc", "itertools 0.14.0", - "jsonschema", "k256", "keyring", "notify", "num-bigint 0.4.6", - "num-integer", "num-traits", "p256", - "pathdiff", "pem", "phf", "pkcs8", "rand 0.10.1", "reqwest", - "schemars", "scrypt", "sec1", "semver", "serde", "serde_cbor", "serde_json", - "serde_yaml", "sha2 0.11.0", - "shellwords", "snafu", "strum 0.28.0", "sysinfo", @@ -3696,7 +3741,6 @@ dependencies = [ "tracing", "url", "uuid", - "wasmparser 0.255.0", "winreg", "wslpath2", "zeroize", @@ -3748,6 +3792,7 @@ dependencies = [ "ic-management-canister-types 0.9.0", "ic-utils", "icp", + "icp-app", "icp-canister-interfaces", "icp-events", "icrc-ledger-types", diff --git a/Cargo.toml b/Cargo.toml index e96fe579c..435e1c938 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,7 @@ ic-ledger-types = "0.16.0" ic-management-canister-types = { version = "0.9.0" } ic-utils = { version = "0.49.1" } icp = { path = "crates/icp" } +icp-app = { path = "crates/icp-app" } icp-canister-interfaces = { path = "crates/icp-canister-interfaces" } icp-events = { path = "crates/icp-events" } icp-sync-plugin = { path = "crates/icp-sync-plugin" } diff --git a/crates/icp-app/Cargo.toml b/crates/icp-app/Cargo.toml new file mode 100644 index 000000000..a343f200d --- /dev/null +++ b/crates/icp-app/Cargo.toml @@ -0,0 +1,86 @@ +[package] +name = "icp-app" +version.workspace = true +edition = { workspace = true } +license = { workspace = true } +publish.workspace = true + +[features] +# `clap::ValueEnum` derives on the settings/identity enums used as CLI value +# types. Enabled by `icp-cli`. +clap = ["dep:clap", "icp/clap"] +# Exposes this crate's mocks, as `icp`'s own feature does for its seams. +test-util = ["icp/test-util"] + +[dependencies] +async-dropper = { workspace = true } +async-trait = { workspace = true } +base64 = { workspace = true } +backoff = { workspace = true } +bigdecimal = { workspace = true } +bip32 = { workspace = true } +bollard = { workspace = true } +camino = { workspace = true } +camino-tempfile = { workspace = true } +candid = { workspace = true } +clap = { workspace = true, optional = true } +crypto-bigint = { workspace = true } +directories = { workspace = true } +dunce = { workspace = true } +ed25519-consensus = { workspace = true } +elliptic-curve = { workspace = true } +flate2 = { workspace = true } +futures = { workspace = true } +hex = { workspace = true } +hmac = { workspace = true } +hybrid-array = { workspace = true } +ic-agent = { workspace = true } +ic-ed25519 = { workspace = true } +ic-identity-hsm = { workspace = true } +ic-ledger-types = { workspace = true } +ic-management-canister-types = { workspace = true } +ic-utils = { workspace = true } +icp = { workspace = true } +icp-canister-interfaces = { workspace = true } +icp-events = { workspace = true } +icrc-ledger-types = { workspace = true } +itertools = { workspace = true } +k256 = { workspace = true } +keyring = { workspace = true } +notify = { workspace = true } +num-bigint = { workspace = true } +num-traits = { workspace = true } +p256 = { workspace = true } +pem = { workspace = true } +phf = { workspace = true } +pkcs8 = { workspace = true } +rand = { workspace = true } +reqwest = { workspace = true } +scrypt = { workspace = true } +sec1 = { workspace = true } +semver = { workspace = true } +serde = { workspace = true } +serde_cbor = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +snafu = { workspace = true } +strum = { workspace = true } +sysinfo = { workspace = true } +tar = { workspace = true } +time = { workspace = true } +tiny-bip39 = { workspace = true } +tokio = { workspace = true, features = ["sync", "macros", "rt", "time", "io-util", "io-std", "process", "signal"] } +tracing = { workspace = true } +url = { workspace = true } +uuid = { workspace = true } +wslpath2 = { workspace = true } +zeroize = { workspace = true } + +[target.'cfg(windows)'.dependencies] +winreg = { workspace = true } + +[dev-dependencies] +httptest = { workspace = true } +icp = { workspace = true, features = ["test-util"] } +indexmap = { workspace = true } +indoc = { workspace = true } diff --git a/crates/icp-app/src/agent.rs b/crates/icp-app/src/agent.rs new file mode 100644 index 000000000..9f9dd5f95 --- /dev/null +++ b/crates/icp-app/src/agent.rs @@ -0,0 +1,70 @@ +use std::{sync::Arc, time::Duration}; + +use async_trait::async_trait; +use ic_agent::{Agent, AgentError, Identity}; +use snafu::prelude::*; + +use icp::prelude::*; + +#[derive(Debug, Snafu)] +pub enum CreateAgentError { + #[snafu(display("failed to create agent"))] + Agent { source: AgentError }, +} + +/// How far ahead of now an agent dates the messages it expires, unless the +/// caller pins something else. +const DEFAULT_INGRESS_EXPIRY: Duration = Duration::from_secs(4 * MINUTE); + +#[async_trait] +pub trait Create: Sync + Send { + /// Builds an agent talking to `url` as `id`. + /// + /// `ingress_expiry` pins how far ahead of now the agent dates the messages it + /// derives an expiry for. Pass `None` for the default. Pass `Some` only when + /// the expiry is itself part of the output — signing a message here for + /// another machine to submit, where the call envelope and the pre-signed + /// `request_status` that accompanies it have to land in the same submission + /// window. A pinned expiry is used verbatim, so the + /// `ICP_CLI_TEST_ADVANCE_TIME_MS` clock offset applies to the default only. + async fn create( + &self, + id: Arc, + url: &str, + ingress_expiry: Option, + ) -> Result; +} + +pub struct Creator; + +#[async_trait] +impl Create for Creator { + async fn create( + &self, + id: Arc, + url: &str, + ingress_expiry: Option, + ) -> Result { + let ingress_expiry = + ingress_expiry.unwrap_or_else(|| DEFAULT_INGRESS_EXPIRY + test_time_advance()); + + let b = Agent::builder() + .with_url(url) + .with_arc_identity(id) + .with_ingress_expiry(ingress_expiry); + + Ok(b.build().context(AgentSnafu)?) + } +} + +/// How far a test has advanced the replica's clock past this machine's, so the +/// default ingress expiry stays ahead of replica time. +fn test_time_advance() -> Duration { + match std::env::var("ICP_CLI_TEST_ADVANCE_TIME_MS") { + Ok(ms) => Duration::from_millis( + ms.parse::() + .expect("ICP_CLI_TEST_ADVANCE_TIME_MS must be set to an int"), + ), + Err(_) => Duration::ZERO, + } +} diff --git a/crates/icp/src/context/init.rs b/crates/icp-app/src/context/init.rs similarity index 82% rename from crates/icp/src/context/init.rs rename to crates/icp-app/src/context/init.rs index a45c4d482..711d28c6d 100644 --- a/crates/icp/src/context/init.rs +++ b/crates/icp-app/src/context/init.rs @@ -2,19 +2,17 @@ use std::{env::current_dir, sync::Arc}; use snafu::prelude::*; -use crate::canister::build::Builder; -use crate::canister::recipe::fetch::RecipeFetcher; -use crate::canister::sync::Syncer; use crate::context::Context; use crate::directories::{Access as _, Directories}; -use crate::prelude::*; -use crate::store_artifact::ArtifactStore; +use crate::recipe::RecipeFetcher; +use icp::canister::build::Builder; +use icp::canister::sync::Syncer; +use icp::prelude::*; +use icp::store_artifact::ArtifactStore; use std::time::Duration; -use crate::{ - Lazy, ProjectLoadImpl, agent, host::Host, identity, identity::PasswordFunc, manifest, network, - store_id, -}; +use crate::{agent, identity, identity::PasswordFunc}; +use icp::{Lazy, ProjectLoadImpl, host::Host, manifest, store_id}; #[derive(Debug, Snafu)] pub enum ContextInitError { @@ -30,10 +28,10 @@ pub enum ContextInitError { Utf8Path { source: FromPathBufError }, #[snafu(display("failed to lock identity directory"))] - IdentityDirectory { source: crate::fs::lock::LockError }, + IdentityDirectory { source: icp::fs::lock::LockError }, #[snafu(display("failed to lock package cache directory"))] - PackageCache { source: crate::fs::lock::LockError }, + PackageCache { source: icp::fs::lock::LockError }, } pub fn initialize( @@ -87,8 +85,16 @@ pub fn initialize( // Prepare http client let http_client = reqwest::Client::new(); - // Package cache - let pkg_cache = dirs.package_cache().context(PackageCacheSnafu)?; + // Package cache. One instance, shared by everything that reads or writes + // it, so the directory lock is taken once per process rather than once per + // holder. + let pkg_cache = Arc::new(dirs.package_cache().context(PackageCacheSnafu)?); + + // Wasm modules named by a manifest but not contained in the project + let wasm = Arc::new(crate::wasm::Fetcher::new( + http_client.clone(), + pkg_cache.clone(), + )); // Recipes let recipe = Arc::new(RecipeFetcher { @@ -97,10 +103,10 @@ pub fn initialize( }); // Canister builder - let builder = Arc::new(Builder); + let builder = Arc::new(Builder::new(wasm.clone())); // Canister syncer - let syncer = Arc::new(Syncer::host()); + let syncer = Arc::new(Syncer::host(wasm.clone())); // Project loader let pload = ProjectLoadImpl { @@ -133,7 +139,7 @@ pub fn initialize( let agent_creator = Arc::new(agent::Creator); // Network accessor - let netaccess = Arc::new(network::Accessor { + let netaccess = Arc::new(crate::network::Accessor { project_root_locate: project_root_locate.clone(), descriptors: dirs.port_descriptor(), agent: agent_creator.clone(), @@ -147,13 +153,16 @@ pub fn initialize( artifacts, builder, syncer, - network: netaccess, - telemetry_data, + wasm, + network: netaccess.clone(), + observer: telemetry_data.clone(), }, dirs, + network_dirs: netaccess.clone(), identity: idload, agent: agent_creator, debug, + telemetry_data, password_func, }) } diff --git a/crates/icp/src/context/mod.rs b/crates/icp-app/src/context/mod.rs similarity index 89% rename from crates/icp/src/context/mod.rs rename to crates/icp-app/src/context/mod.rs index 96eed7386..599ed4c24 100644 --- a/crates/icp/src/context/mod.rs +++ b/crates/icp-app/src/context/mod.rs @@ -2,20 +2,18 @@ use std::sync::Arc; use url::Url; use crate::{ - agent::CreateAgentError, - directories, + agent::CreateAgentError, directories, identity::IdentitySelection, telemetry_data::NetworkType, +}; +use candid::Principal; +use ic_agent::{Agent, Identity}; +use icp::{ host::{ CanisterSelection, EnvironmentSelection, GetCanisterIdForEnvError, GetEnvironmentError, Host, }, - identity::IdentitySelection, - manifest::network::RootKeySpec, - network::{Configuration as NetworkConfiguration, access::NetworkAccess}, + network::{Configuration as NetworkConfiguration, NetworkAccess, RootKeySpec}, prelude::*, - telemetry_data::NetworkType, }; -use candid::Principal; -use ic_agent::{Agent, Identity}; use snafu::{OptionExt, ResultExt, Snafu}; use time::OffsetDateTime; @@ -23,8 +21,6 @@ mod init; pub use init::initialize; -pub const IC_ROOT_KEY: &[u8; 133] = b"\x30\x81\x82\x30\x1d\x06\x0d\x2b\x06\x01\x04\x01\x82\xdc\x7c\x05\x03\x01\x02\x01\x06\x0c\x2b\x06\x01\x04\x01\x82\xdc\x7c\x05\x03\x02\x01\x03\x61\x00\x81\x4c\x0e\x6e\xc7\x1f\xab\x58\x3b\x08\xbd\x81\x37\x3c\x25\x5c\x3c\x37\x1b\x2e\x84\x86\x3c\x98\xa4\xf1\xe0\x8b\x74\x23\x5d\x14\xfb\x5d\x9c\x0c\xd5\x46\xd9\x68\x5f\x91\x3a\x0c\x0b\x2c\xc5\x34\x15\x83\xbf\x4b\x43\x92\xe4\x67\xdb\x96\xd6\x5b\x9b\xb4\xcb\x71\x71\x12\xf8\x47\x2e\x0d\x5a\x4d\x14\x50\x5f\xfd\x74\x84\xb0\x12\x91\x09\x1c\x5f\x87\xb9\x88\x83\x46\x3f\x98\x09\x1a\x0b\xaa\xae"; - /// Selection type for networks - similar to IdentitySelection #[derive(Clone, Debug, PartialEq)] pub enum NetworkSelection { @@ -53,6 +49,10 @@ pub struct Context { /// Various cli-related directories (cache, configuration, etc). pub dirs: Arc, + /// Where a network keeps its on-disk state. Not part of + /// [`icp::network::Access`] because the layout is this crate's invention. + pub network_dirs: Arc, + /// Identity loader identity: Arc, @@ -62,6 +62,9 @@ pub struct Context { /// Whether debug is enabled pub debug: bool, + /// Telemetry data collected during command execution + pub telemetry_data: Arc, + /// Password reader for identity decryption; shared with the identity loader. pub password_func: Arc Result + Send + Sync>, } @@ -90,7 +93,7 @@ impl Context { pub async fn get_network( &self, network_selection: &NetworkSelection, - ) -> Result { + ) -> Result { let network = match network_selection { NetworkSelection::Named(network_name) => { if self.host.project.exists().await? { @@ -100,10 +103,10 @@ impl Context { })?; net.clone() } else if network_name == IC { - crate::Network { + icp::Network { name: IC.to_string(), - configuration: crate::network::Configuration::Connected { - connected: crate::network::Connected { + configuration: icp::network::Configuration::Connected { + connected: icp::network::Connected { api_url: IC_MAINNET_NETWORK_API_URL.parse().unwrap(), http_gateway_url: Some( IC_MAINNET_NETWORK_GATEWAY_URL.parse().unwrap(), @@ -119,10 +122,10 @@ impl Context { } } NetworkSelection::Default => return Err(GetNetworkError::DefaultNetwork), - NetworkSelection::Url(url, root_key) => crate::Network { + NetworkSelection::Url(url, root_key) => icp::Network { name: url.to_string(), - configuration: crate::network::Configuration::Connected { - connected: crate::network::Connected { + configuration: icp::network::Configuration::Connected { + connected: icp::network::Connected { api_url: url.clone(), http_gateway_url: Some(url.clone()), root_key: root_key.clone(), @@ -135,7 +138,7 @@ impl Context { NetworkConfiguration::Managed { .. } => NetworkType::Managed, NetworkConfiguration::Connected { .. } => NetworkType::Connected, }; - self.host.telemetry_data.set_network_type(network_type); + self.telemetry_data.set_network_type(network_type); Ok(network) } @@ -148,7 +151,7 @@ impl Context { pub async fn get_network_or_environment( &self, selection: &NetworkOrEnvironmentSelection, - ) -> Result { + ) -> Result { match selection { NetworkOrEnvironmentSelection::Network(network_name) => { let network_selection = NetworkSelection::Named(network_name.clone()); @@ -262,7 +265,7 @@ impl Context { Err(GetAgentForEnvError::GetEnvironment { source: GetEnvironmentError::ProjectLoad { - source: crate::ProjectLoadError::Locate { .. }, + source: icp::ProjectLoadError::Locate { .. }, }, }) => Err(GetAgentError::NoProjectOrNetwork), Err(e) => Err(e.into()), @@ -319,7 +322,9 @@ impl Context { pub fn mocked() -> Context { Context { host: Host::mocked(), + telemetry_data: Arc::new(crate::telemetry_data::TelemetryData::default()), dirs: Arc::new(crate::directories::UnimplementedMockDirs), + network_dirs: Arc::new(crate::network::UnimplementedMockDirectories), identity: Arc::new(crate::identity::MockIdentityLoader::anonymous()), agent: Arc::new(crate::agent::Creator), debug: false, @@ -340,7 +345,7 @@ pub enum GetIdentityError { #[derive(Debug, Snafu)] pub enum GetNetworkError { #[snafu(transparent)] - ProjectLoad { source: crate::ProjectLoadError }, + ProjectLoad { source: icp::ProjectLoadError }, #[snafu(display("project does not contain a network named '{}'", name))] NetworkNotFound { name: String }, @@ -370,7 +375,7 @@ pub enum GetAgentForEnvError { GetEnvironment { source: GetEnvironmentError }, #[snafu(transparent)] - NetworkAccess { source: crate::network::AccessError }, + NetworkAccess { source: icp::network::AccessError }, #[snafu(transparent)] AgentCreate { @@ -387,7 +392,7 @@ pub enum GetAgentForNetworkError { GetNetwork { source: GetNetworkError }, #[snafu(transparent)] - NetworkAccess { source: crate::network::AccessError }, + NetworkAccess { source: icp::network::AccessError }, #[snafu(transparent)] AgentCreate { @@ -425,7 +430,7 @@ pub enum GetAgentForSigningError { #[derive(Debug, Snafu)] pub enum GetAgentError { #[snafu(transparent)] - ProjectExists { source: crate::ProjectLoadError }, + ProjectExists { source: icp::ProjectLoadError }, #[snafu(display("You can't specify both an environment and a network"))] EnvironmentAndNetworkSpecified, diff --git a/crates/icp/src/context/tests.rs b/crates/icp-app/src/context/tests.rs similarity index 94% rename from crates/icp/src/context/tests.rs rename to crates/icp-app/src/context/tests.rs index 788c0cb4c..a597bfeaa 100644 --- a/crates/icp/src/context/tests.rs +++ b/crates/icp-app/src/context/tests.rs @@ -1,15 +1,16 @@ use super::*; -use crate::{ +use crate::identity::MockIdentityLoader; +use candid::Principal; +use icp::network::MockNetworkAccessor; +use icp::{ Environment, MockProjectLoader, Network, Project, host::SetCanisterIdForEnvError, - identity::MockIdentityLoader, network::{ - Configuration, Gateway, Managed, ManagedLauncherConfig, ManagedMode, MockNetworkAccessor, - Port, access::NetworkAccess, + Configuration, Gateway, Managed, ManagedLauncherConfig, ManagedMode, Port, + access::NetworkAccess, }, store_id::{Access as IdAccess, mock::MockInMemoryIdStore}, }; -use candid::Principal; use indexmap::IndexMap; use std::collections::HashMap; @@ -353,7 +354,7 @@ async fn test_remove_canister_id_for_env_success() { let lookup_result = ids_store.lookup(true, "dev", "backend"); assert!(matches!( lookup_result, - Err(crate::store_id::LookupIdError::IdNotFound { .. }) + Err(icp::store_id::LookupIdError::IdNotFound { .. }) )); } @@ -392,7 +393,7 @@ async fn test_get_agent_for_env_uses_environment_network() { "local", NetworkAccess { root_key: local_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse("http://localhost:8000").unwrap(), http_gateway_url: None, use_friendly_domains: false, @@ -402,7 +403,7 @@ async fn test_get_agent_for_env_uses_environment_network() { "staging", NetworkAccess { root_key: staging_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse("http://staging:9000").unwrap(), http_gateway_url: None, use_friendly_domains: false, @@ -466,9 +467,7 @@ async fn test_get_agent_for_env_network_not_configured() { assert!(matches!( result, - Err(GetAgentForEnvError::NetworkAccess { - source: crate::network::AccessError::GetNetworkAccess { .. } - }) + Err(GetAgentForEnvError::NetworkAccess { .. }) )); } @@ -483,7 +482,7 @@ async fn test_get_agent_for_network_success() { "local", NetworkAccess { root_key: root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse("http://localhost:8000").unwrap(), http_gateway_url: None, use_friendly_domains: false, @@ -545,9 +544,7 @@ async fn test_get_agent_for_network_not_configured() { assert!(matches!( result, - Err(GetAgentForNetworkError::NetworkAccess { - source: crate::network::AccessError::GetNetworkAccess { .. } - }) + Err(GetAgentForNetworkError::NetworkAccess { .. }) )); } @@ -642,7 +639,7 @@ async fn test_ids_by_environment() { async fn test_get_agent_defaults_outside_project() { let ctx = Context { host: Host { - project: Arc::new(crate::NoProjectLoader), + project: Arc::new(icp::NoProjectLoader), ..Host::mocked() }, ..Context::mocked() @@ -711,12 +708,12 @@ async fn test_get_agent_defaults_inside_project_with_default_local() { let ctx = Context { host: Host { - project: Arc::new(crate::MockProjectLoader::new(project)), + project: Arc::new(icp::MockProjectLoader::new(project)), network: Arc::new(MockNetworkAccessor::new().with_network( LOCAL, NetworkAccess { root_key: local_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse(DEFAULT_LOCAL_NETWORK_URL).unwrap(), http_gateway_url: None, use_friendly_domains: false, @@ -789,12 +786,12 @@ async fn test_get_agent_defaults_with_overridden_local_network() { let ctx = Context { host: Host { - project: Arc::new(crate::MockProjectLoader::new(project)), + project: Arc::new(icp::MockProjectLoader::new(project)), network: Arc::new(MockNetworkAccessor::new().with_network( LOCAL, NetworkAccess { root_key: custom_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse("http://localhost:9000").unwrap(), // Custom port http_gateway_url: None, use_friendly_domains: false, @@ -892,14 +889,14 @@ async fn test_get_agent_defaults_with_overridden_local_environment() { let ctx = Context { host: Host { - project: Arc::new(crate::MockProjectLoader::new(project)), + project: Arc::new(icp::MockProjectLoader::new(project)), network: Arc::new( MockNetworkAccessor::new() .with_network( LOCAL, NetworkAccess { root_key: local_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse(DEFAULT_LOCAL_NETWORK_URL).unwrap(), http_gateway_url: None, use_friendly_domains: false, @@ -909,7 +906,7 @@ async fn test_get_agent_defaults_with_overridden_local_environment() { "custom", NetworkAccess { root_key: custom_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse("http://localhost:7000").unwrap(), http_gateway_url: None, use_friendly_domains: false, @@ -948,7 +945,7 @@ async fn test_get_agent_explicit_network_inside_project() { LOCAL, NetworkAccess { root_key: local_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse(DEFAULT_LOCAL_NETWORK_URL).unwrap(), http_gateway_url: None, use_friendly_domains: false, @@ -958,7 +955,7 @@ async fn test_get_agent_explicit_network_inside_project() { "staging", NetworkAccess { root_key: staging_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse("http://localhost:8001").unwrap(), http_gateway_url: None, use_friendly_domains: false, @@ -998,7 +995,7 @@ async fn test_get_agent_explicit_environment_inside_project() { LOCAL, NetworkAccess { root_key: local_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse(DEFAULT_LOCAL_NETWORK_URL).unwrap(), http_gateway_url: None, use_friendly_domains: false, @@ -1008,7 +1005,7 @@ async fn test_get_agent_explicit_environment_inside_project() { "staging", NetworkAccess { root_key: staging_root_key.clone(), - root_key_source: crate::network::RootKeySource::Configured, + root_key_source: icp::network::RootKeySource::Configured, api_url: Url::parse("http://localhost:8001").unwrap(), http_gateway_url: None, use_friendly_domains: false, diff --git a/crates/icp/src/directories.rs b/crates/icp-app/src/directories.rs similarity index 98% rename from crates/icp/src/directories.rs rename to crates/icp-app/src/directories.rs index 37f9f2724..607a99bd3 100644 --- a/crates/icp/src/directories.rs +++ b/crates/icp-app/src/directories.rs @@ -5,13 +5,12 @@ //! custom overrides, primarily for storing user data like identities and cache. use crate::{ - fs::lock::LockError, identity::{IdentityDirectories, IdentityPaths}, package::PackageCache, - prelude::*, settings::{SettingsDirectories, SettingsPaths}, }; use directories::ProjectDirs; +use icp::{fs::lock::LockError, prelude::*}; use snafu::prelude::*; /// Trait for accessing global ICP CLI directories. @@ -193,7 +192,7 @@ impl Access for Directories { /// /// This directory stores downloaded versions of managed packages like icp-cli-network-launcher. fn package_cache(&self) -> Result { - PackageCache::new(self.data_local().join("pkg")) + crate::package::open(self.data_local().join("pkg")) } /// Returns the path to the user settings directory. diff --git a/crates/icp/src/identity/delegation.rs b/crates/icp-app/src/identity/delegation.rs similarity index 99% rename from crates/icp/src/identity/delegation.rs rename to crates/icp-app/src/identity/delegation.rs index 9ac9e68f2..64833624d 100644 --- a/crates/icp/src/identity/delegation.rs +++ b/crates/icp-app/src/identity/delegation.rs @@ -5,7 +5,7 @@ use ic_agent::export::Principal; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; -use crate::{fs, prelude::*}; +use icp::{fs, prelude::*}; /// Matches the Candid `DelegationChain` record from the cli-backend canister. /// All byte fields are hex-encoded strings on the wire. diff --git a/crates/icp/src/identity/key.rs b/crates/icp-app/src/identity/key.rs similarity index 98% rename from crates/icp/src/identity/key.rs rename to crates/icp-app/src/identity/key.rs index 3c28075e0..468927f67 100644 --- a/crates/icp/src/identity/key.rs +++ b/crates/icp-app/src/identity/key.rs @@ -27,20 +27,19 @@ use tracing::{debug, warn}; use url::Url; use zeroize::Zeroizing; -use crate::{ - context::IC_ROOT_KEY, +use crate::identity::{ + IdentityPaths, PasswordFunc, + delegation::{self, SignedDelegation}, + manifest::{ + DelegationKeyStorage, IdentityDefaults, IdentityKeyAlgorithm, IdentityList, IdentitySpec, + LoadIdentityManifestError, PemFormat, WriteIdentityManifestError, + }, +}; +use icp::{ fs::{ self, lock::{LRead, LWrite}, }, - identity::{ - IdentityPaths, PasswordFunc, - delegation::{self, SignedDelegation}, - manifest::{ - DelegationKeyStorage, IdentityDefaults, IdentityKeyAlgorithm, IdentityList, - IdentitySpec, LoadIdentityManifestError, PemFormat, WriteIdentityManifestError, - }, - }, prelude::*, }; @@ -67,7 +66,7 @@ pub enum ExportFormat { #[derive(Debug, Snafu)] pub enum LoadIdentityError { #[snafu(transparent)] - ReadFileError { source: crate::fs::IoError }, + ReadFileError { source: icp::fs::IoError }, #[snafu(display("failed to load PEM from `{origin}`: failed to parse"))] ParsePemError { @@ -100,7 +99,7 @@ pub enum LoadIdentityError { GetPasswordError { message: String }, #[snafu(transparent)] - LockError { source: crate::fs::lock::LockError }, + LockError { source: icp::fs::lock::LockError }, #[snafu(display("failed to load keyring entry"))] LoadEntryError { source: keyring::Error }, @@ -404,7 +403,7 @@ fn try_load_pem_session(dirs: LRead<&IdentityPaths>, name: &str) -> Option Result { - crate::fs::create_dir_all(&self.dir)?; + pub fn ensure_identity_defaults_path(&self) -> Result { + icp::fs::create_dir_all(&self.dir)?; Ok(self.dir.join(IDENTITY_DEFAULTS)) } @@ -53,8 +55,8 @@ impl IdentityPaths { self.dir.join(IDENTITIES_LIST) } - pub fn ensure_identity_list_path(&self) -> Result { - crate::fs::create_dir_all(&self.dir)?; + pub fn ensure_identity_list_path(&self) -> Result { + icp::fs::create_dir_all(&self.dir)?; Ok(self.dir.join(IDENTITIES_LIST)) } @@ -62,8 +64,8 @@ impl IdentityPaths { self.dir.join(format!("keys/{name}.pem")) } - pub fn ensure_key_pem_path(&self, name: &str) -> Result { - crate::fs::create_dir_all(&self.dir.join("keys"))?; + pub fn ensure_key_pem_path(&self, name: &str) -> Result { + icp::fs::create_dir_all(&self.dir.join("keys"))?; Ok(self.dir.join(format!("keys/{name}.pem"))) } @@ -71,8 +73,8 @@ impl IdentityPaths { self.dir.join(format!("delegations/{name}.json")) } - pub fn ensure_delegation_chain_path(&self, name: &str) -> Result { - crate::fs::create_dir_all(&self.dir.join("delegations"))?; + pub fn ensure_delegation_chain_path(&self, name: &str) -> Result { + icp::fs::create_dir_all(&self.dir.join("delegations"))?; Ok(self.dir.join(format!("delegations/{name}.json"))) } } diff --git a/crates/icp/src/identity/seed/mod.rs b/crates/icp-app/src/identity/seed/mod.rs similarity index 100% rename from crates/icp/src/identity/seed/mod.rs rename to crates/icp-app/src/identity/seed/mod.rs diff --git a/crates/icp/src/identity/seed/slip10.rs b/crates/icp-app/src/identity/seed/slip10.rs similarity index 100% rename from crates/icp/src/identity/seed/slip10.rs rename to crates/icp-app/src/identity/seed/slip10.rs diff --git a/crates/icp-app/src/lib.rs b/crates/icp-app/src/lib.rs new file mode 100644 index 000000000..cc01acdc8 --- /dev/null +++ b/crates/icp-app/src/lib.rs @@ -0,0 +1,27 @@ +//! Everything about the machine the tool is running on, rather than about a +//! project. +//! +//! Identities and the keyring, user settings, the global directory layout, the +//! package cache, local networks and the launcher that runs them, telemetry, +//! and the operations that act on a canister by principal rather than by what +//! some manifest says about it. +//! +//! Projects — manifests, building, installing, syncing, deploying — are +//! [`icp`], which this crate depends on and which does not depend on this one. +//! The seams that project code reaches the machine through +//! ([`icp::network::Access`], [`icp::canister::wasm::Fetch`], +//! [`icp::canister::recipe::Resolve`], [`icp::host::Observe`]) are declared +//! there and implemented here. + +pub mod agent; +pub mod context; +pub mod directories; +pub mod identity; +pub mod network; +pub mod operations; +pub mod package; +pub mod recipe; +pub mod settings; +pub mod signed_message; +pub mod telemetry_data; +pub mod wasm; diff --git a/crates/icp-app/src/network/accessor.rs b/crates/icp-app/src/network/accessor.rs new file mode 100644 index 000000000..c67efb343 --- /dev/null +++ b/crates/icp-app/src/network/accessor.rs @@ -0,0 +1,180 @@ +//! The host's answer to [`icp::network::Access`]: resolving a project's +//! networks against what is actually running on this machine. + +use std::{collections::BTreeMap, sync::Arc}; + +use async_trait::async_trait; +use candid::Principal; +use snafu::{ResultExt, Snafu}; +use url::Url; + +use icp::manifest::{ProjectRootLocate, ProjectRootLocateError}; +use icp::network::{ + Access, AccessError, CollectFriendlyDomains, Configuration, NetworkAccess, NetworkUrls, +}; +use icp::prelude::*; +use icp::{CACHE_DIR, ICP_BASE, Network}; + +use crate::network::{ + NetworkDirectory, custom_domains, + resolve::{get_connected_network_access, get_managed_network_access, get_managed_network_urls}, +}; + +/// Locating a network's directory needs the project root, which may not be +/// there at all. +#[derive(Debug, Snafu)] +pub enum LocateNetworkDirectoryError { + #[snafu(display("failed to find project root"))] + ProjectRootLocate { source: ProjectRootLocateError }, +} + +/// Where a network keeps its on-disk state. +/// +/// Separate from [`Access`] because the directory layout is this crate's +/// invention: the project layer never needs to know a network has one. +pub trait Directories: Send + Sync { + fn get_network_directory( + &self, + network: &Network, + ) -> Result; +} + +#[cfg(any(test, feature = "test-util"))] +/// A [`Directories`] for tests that never look one up. +pub struct UnimplementedMockDirectories; + +#[cfg(any(test, feature = "test-util"))] +impl Directories for UnimplementedMockDirectories { + fn get_network_directory( + &self, + _network: &Network, + ) -> Result { + unimplemented!("UnimplementedMockDirectories::get_network_directory") + } +} + +pub struct Accessor { + // Project root + pub project_root_locate: Arc, + + // Port descriptors dir + pub descriptors: PathBuf, + + // Used to build a bootstrap agent when a connected network fetches its root key + pub agent: Arc, +} + +impl Accessor { + /// The network directory is located at `/.icp/cache/networks/`. + pub fn get_network_directory( + &self, + network: &Network, + ) -> Result { + let dir = self + .project_root_locate + .locate() + .context(ProjectRootLocateSnafu)?; + Ok(NetworkDirectory::new( + &network.name, + &dir.join(ICP_BASE) + .join(CACHE_DIR) + .join("networks") + .join(&network.name), + &self.descriptors, + )) + } +} + +#[async_trait] +impl Directories for Accessor { + fn get_network_directory( + &self, + network: &Network, + ) -> Result { + Accessor::get_network_directory(self, network) + } +} + +#[async_trait] +impl Access for Accessor { + async fn access(&self, network: &Network) -> Result { + match &network.configuration { + Configuration::Managed { managed: _ } => { + let nd = self + .get_network_directory(network) + .map_err(AccessError::new)?; + get_managed_network_access(nd) + .await + .map_err(AccessError::new) + } + Configuration::Connected { connected: cfg } => { + get_connected_network_access(cfg, &self.agent) + .await + .map_err(AccessError::new) + } + } + } + + async fn urls(&self, network: &Network) -> Result { + match &network.configuration { + Configuration::Managed { managed: _ } => { + let nd = self + .get_network_directory(network) + .map_err(AccessError::new)?; + get_managed_network_urls(nd).await.map_err(AccessError::new) + } + // A connected network's endpoints are configured, so there is + // nothing to resolve. + Configuration::Connected { connected: cfg } => Ok(NetworkUrls { + api_url: cfg.api_url.clone(), + http_gateway_url: cfg.http_gateway_url.clone(), + }), + } + } + + async fn publish_friendly_domains( + &self, + network: &Network, + collect: &CollectFriendlyDomains<'_>, + ) { + let Configuration::Managed { .. } = &network.configuration else { + return; + }; + let Ok(nd) = self.get_network_directory(network) else { + return; + }; + let Ok(Some(desc)) = nd.load_network_descriptor().await else { + return; + }; + let Some(status_dir) = &desc.status_dir else { + return; + }; + let gateway_url_str = format!("http://{}:{}", desc.gateway.host, desc.gateway.port); + let Ok(gateway_url) = Url::parse(&gateway_url_str) else { + tracing::warn!("Failed to parse gateway URL {gateway_url_str:?} for custom domains"); + return; + }; + let Some(domain) = custom_domains::gateway_domain(&gateway_url) else { + return; + }; + + // Only here, past every way this can turn out to have nothing to write, + // is the project asked for any mappings. The descriptor names the + // network the gateway is actually serving, so it — not the + // environment's own view — decides which environments share this + // network and therefore this mapping file. + let env_entries: BTreeMap> = collect(&desc.network) + .into_iter() + .map(|e| (e.environment, e.entries)) + .collect(); + + let extra: Vec<_> = custom_domains::ii_custom_domain_entry(desc.ii, domain) + .into_iter() + .collect(); + if let Err(e) = + custom_domains::write_custom_domains(status_dir, domain, &env_entries, &extra) + { + tracing::warn!("Failed to update custom domains: {e}"); + } + } +} diff --git a/crates/icp/src/network/config.rs b/crates/icp-app/src/network/config.rs similarity index 99% rename from crates/icp/src/network/config.rs rename to crates/icp-app/src/network/config.rs index d8cb1be3e..7894a2655 100644 --- a/crates/icp/src/network/config.rs +++ b/crates/icp-app/src/network/config.rs @@ -18,7 +18,7 @@ use snafu::prelude::*; use url::Url; use uuid::Uuid; -use crate::prelude::*; +use icp::prelude::*; /// How long to wait for the gateway to answer before concluding the network is defunct. const PROBE_TIMEOUT: Duration = Duration::from_secs(5); diff --git a/crates/icp/src/network/custom_domains.rs b/crates/icp-app/src/network/custom_domains.rs similarity index 99% rename from crates/icp/src/network/custom_domains.rs rename to crates/icp-app/src/network/custom_domains.rs index 8bfbd2d26..18695b673 100644 --- a/crates/icp/src/network/custom_domains.rs +++ b/crates/icp-app/src/network/custom_domains.rs @@ -4,7 +4,7 @@ use candid::Principal; use snafu::prelude::*; use url::Url; -use crate::prelude::*; +use icp::prelude::*; /// Writes a `custom-domains.txt` file to the given status directory. /// @@ -40,7 +40,7 @@ pub fn write_custom_domains( for (full_domain, canister_id) in extra_entries { content.push_str(&format!("{full_domain}:{canister_id}\n")); } - crate::fs::write(&file_path, content.as_bytes())?; + icp::fs::write(&file_path, content.as_bytes())?; Ok(()) } @@ -143,7 +143,7 @@ pub fn canister_gateway_url( #[derive(Debug, Snafu)] pub enum WriteCustomDomainsError { #[snafu(transparent)] - WriteFile { source: crate::fs::IoError }, + WriteFile { source: icp::fs::IoError }, } #[cfg(test)] diff --git a/crates/icp/src/network/directory.rs b/crates/icp-app/src/network/directory.rs similarity index 96% rename from crates/icp/src/network/directory.rs rename to crates/icp-app/src/network/directory.rs index 3549d6b42..873502c2e 100644 --- a/crates/icp/src/network/directory.rs +++ b/crates/icp-app/src/network/directory.rs @@ -53,12 +53,12 @@ use std::io::ErrorKind; use snafu::{ResultExt, prelude::*}; -use crate::{ +use crate::network::config::NetworkDescriptorModel; +use icp::{ fs::{ create_dir_all, json, lock::{DirectoryStructureLock, LWrite, LockError, PathsAccess}, }, - network::config::NetworkDescriptorModel, prelude::*, }; @@ -100,7 +100,7 @@ pub enum LoadNetworkFileError { } impl NetworkDirectory { - pub fn ensure_exists(&self) -> Result<(), crate::fs::IoError> { + pub fn ensure_exists(&self) -> Result<(), icp::fs::IoError> { // Network root create_dir_all(&self.network_root)?; @@ -150,7 +150,7 @@ impl NetworkDirectory { &self, ) -> Result<(), CleanupNetworkDescriptorError> { self.root()? - .with_write(async |root| crate::fs::remove_file(&root.network_descriptor_path())) + .with_write(async |root| icp::fs::remove_file(&root.network_descriptor_path())) .await??; Ok(()) } @@ -162,7 +162,7 @@ impl NetworkDirectory { ) -> Result<(), CleanupNetworkDescriptorError> { if let Some(port) = gateway_port { self.port(port)? - .with_write(async |paths| crate::fs::remove_file(&paths.descriptor_path())) + .with_write(async |paths| icp::fs::remove_file(&paths.descriptor_path())) .await??; } Ok(()) @@ -308,7 +308,7 @@ pub enum CleanupNetworkDescriptorError { #[snafu(transparent)] LockFileError { source: LockError }, #[snafu(transparent)] - DeleteFileError { source: crate::fs::IoError }, + DeleteFileError { source: icp::fs::IoError }, } #[derive(Debug, Snafu)] @@ -317,14 +317,14 @@ pub enum SavePidError { LockFileError { source: LockError }, #[snafu(transparent)] - WritePid { source: crate::fs::IoError }, + WritePid { source: icp::fs::IoError }, } #[derive(Debug, Snafu)] pub enum LoadPidError { #[snafu(display("failed to read PID from {path}"))] ReadPid { - source: crate::fs::IoError, + source: icp::fs::IoError, path: PathBuf, }, #[snafu(transparent)] diff --git a/crates/icp/src/network/managed/cache.rs b/crates/icp-app/src/network/managed/cache.rs similarity index 92% rename from crates/icp/src/network/managed/cache.rs rename to crates/icp-app/src/network/managed/cache.rs index cf70963ae..e511bc7b8 100644 --- a/crates/icp/src/network/managed/cache.rs +++ b/crates/icp-app/src/network/managed/cache.rs @@ -8,9 +8,9 @@ use snafu::{ResultExt, Snafu}; use tar::Archive; use tracing::debug; -use crate::fs::lock::{LRead, LWrite}; use crate::package::{PackageCachePaths, get_tag, get_tag_with_updater, set_tag_with_updater}; -use crate::prelude::*; +use icp::fs::lock::{LRead, LWrite}; +use icp::prelude::*; const LAUNCHER_NAME: &str = "icp-cli-network-launcher"; @@ -85,7 +85,7 @@ fn is_updater_stale(updater_version: Option<&str>) -> bool { #[derive(Debug, Snafu)] pub enum ReadCacheError { #[snafu(display("failed to read package tag"))] - LoadTag { source: crate::fs::json::Error }, + LoadTag { source: icp::fs::json::Error }, } pub async fn get_latest_launcher_version(client: &Client) -> Result { @@ -126,7 +126,7 @@ pub async fn download_launcher_version( version_req.to_owned() }; let version_path = paths.launcher_version(&pkg_version); - crate::fs::create_dir_all(&paths.launcher_dir()).context(CreateDirSnafu)?; + icp::fs::create_dir_all(&paths.launcher_dir()).context(CreateDirSnafu)?; let mut tmp = camino_tempfile::tempfile().context(TempFileSnafu)?; let tmp_write = BufWriter::new(&tmp); let arch = match std::env::consts::ARCH { @@ -171,17 +171,17 @@ pub async fn download_launcher_version( let decompressor = GzDecoder::new(tmp_read); let mut archive = Archive::new(decompressor); let extract_dir = paths.launcher_dir().join("tmp"); - crate::fs::create_dir_all(&extract_dir).context(TempDirSnafu)?; + icp::fs::create_dir_all(&extract_dir).context(TempDirSnafu)?; let tarball_name = format!("icp-cli-network-launcher-{arch}-{os}-{pkg_version}"); let extracted_dir_path = extract_dir.join(&tarball_name); if extracted_dir_path.exists() { - crate::fs::remove_dir_all(&extracted_dir_path).context(RemoveExistingSnafu)? + icp::fs::remove_dir_all(&extracted_dir_path).context(RemoveExistingSnafu)? } archive .unpack(&extract_dir) .context(ExtractSnafu { path: &extract_dir })?; if version_path.exists() { - crate::fs::remove_dir_all(&version_path).context(RemoveExistingSnafu)? + icp::fs::remove_dir_all(&version_path).context(RemoveExistingSnafu)? } std::fs::rename(&extracted_dir_path, &version_path).context(MoveExtractedSnafu { from: extracted_dir_path, @@ -201,7 +201,7 @@ pub async fn check_launcher_update_available( client: &Client, ) -> Option { let ts_path = paths.update_nag_timestamp(); - if let Ok(contents) = crate::fs::read_to_string(&ts_path) + if let Ok(contents) = icp::fs::read_to_string(&ts_path) && let Ok(ts) = contents.trim().parse::() { let then = SystemTime::UNIX_EPOCH + Duration::from_secs(ts); @@ -216,7 +216,7 @@ pub async fn check_launcher_update_available( .expect("since epoch") .as_secs(); // Write timestamp regardless of outcome, so we don't re-check on failure - let _ = crate::fs::write(&ts_path, format!("{now}\n").as_bytes()); + let _ = icp::fs::write(&ts_path, format!("{now}\n").as_bytes()); let latest = get_latest_launcher_version(client).await.ok()?; if latest != cached_version { @@ -247,11 +247,11 @@ pub enum DownloadLauncherError { #[snafu(display("failed to save downloaded network launcher"))] SaveDownload { source: std::io::Error }, #[snafu(display("failed to remove existing launcher"))] - RemoveExisting { source: crate::fs::IoError }, + RemoveExisting { source: icp::fs::IoError }, #[snafu(display("failed to create temporary file for download"))] TempFile { source: std::io::Error }, #[snafu(display("failed to create temporary directory for extraction"))] - TempDir { source: crate::fs::IoError }, + TempDir { source: icp::fs::IoError }, #[snafu(display("buffer failure in temporary file"))] Buffer { source: std::io::Error }, #[snafu(display("failed to extract downloaded network launcher to {path}"))] @@ -266,11 +266,11 @@ pub enum DownloadLauncherError { to: PathBuf, }, #[snafu(display("failed to create network launcher cache directory"))] - CreateDir { source: crate::fs::IoError }, + CreateDir { source: icp::fs::IoError }, #[snafu(display("failed to fetch latest network launcher version from GitHub"))] LatestVersionFetch { source: reqwest::Error }, #[snafu(display("failed to parse latest version response from GitHub"))] LatestVersionParse, #[snafu(display("failed to create package tag"))] - CreateTag { source: crate::fs::json::Error }, + CreateTag { source: icp::fs::json::Error }, } diff --git a/crates/icp/src/network/managed/docker.rs b/crates/icp-app/src/network/managed/docker.rs similarity index 99% rename from crates/icp/src/network/managed/docker.rs rename to crates/icp-app/src/network/managed/docker.rs index 581a29434..2b6c45e79 100644 --- a/crates/icp/src/network/managed/docker.rs +++ b/crates/icp-app/src/network/managed/docker.rs @@ -21,11 +21,11 @@ use tracing::{debug, info}; use wslpath2::Conversion; use crate::network::{ - ManagedImageConfig, config::ChildLocator, managed::launcher::{CUSTOM_DOMAINS_FEATURE, NetworkInstance}, }; -use crate::prelude::*; +use icp::network::ManagedImageConfig; +use icp::prelude::*; use super::launcher::{ MAX_OUTPUT_TAIL_BYTES, MAX_OUTPUT_TAIL_LINES, output_tail, wait_for_launcher_status, diff --git a/crates/icp/src/network/managed/launcher.rs b/crates/icp-app/src/network/managed/launcher.rs similarity index 98% rename from crates/icp/src/network/managed/launcher.rs rename to crates/icp-app/src/network/managed/launcher.rs index 38d27d65f..d2f24b43d 100644 --- a/crates/icp/src/network/managed/launcher.rs +++ b/crates/icp-app/src/network/managed/launcher.rs @@ -9,8 +9,9 @@ use sysinfo::{Pid, ProcessesToUpdate, Signal, System}; use tokio::{process::Child, select, sync::mpsc::Sender, time::Instant}; use tracing::{info, warn}; -use crate::{ - network::{ManagedLauncherConfig, Port, config::ChildLocator}, +use crate::network::config::ChildLocator; +use icp::{ + network::{ManagedLauncherConfig, Port}, prelude::*, }; @@ -177,7 +178,7 @@ fn premature_exit_detail(background: bool, stderr_file: &Path) -> String { if !background { return String::new(); } - match crate::fs::read_to_string(stderr_file) { + match icp::fs::read_to_string(stderr_file) { Ok(contents) => { let tail = output_tail(&contents); if tail.is_empty() { @@ -333,7 +334,7 @@ pub enum WaitForFileError { }, #[snafu(transparent)] - ReadFile { source: crate::fs::IoError }, + ReadFile { source: icp::fs::IoError }, } /// Waits for a file to be created and have a full line of content. Call the function before initing the external process, @@ -377,7 +378,7 @@ pub fn wait_for_single_line_file( }; let event = res.context(ReadEventSnafu { path: &dir })?; if event.kind.is_modify() || event.kind.is_create() { - match crate::fs::read_to_string(&path) { + match icp::fs::read_to_string(&path) { Ok(content) => { if content.ends_with('\n') { return Ok(content); @@ -505,7 +506,7 @@ mod tests { fn premature_exit_detail_includes_captured_output() { let dir = camino_tempfile::Utf8TempDir::new().unwrap(); let file = dir.path().join("stderr.log"); - crate::fs::write(&file, b"Address already in use (os error 48)\n").unwrap(); + icp::fs::write(&file, b"Address already in use (os error 48)\n").unwrap(); let detail = premature_exit_detail(true, &file); assert!(detail.starts_with('\n')); assert!(detail.contains("Address already in use")); diff --git a/crates/icp/src/network/managed/mod.rs b/crates/icp-app/src/network/managed/mod.rs similarity index 100% rename from crates/icp/src/network/managed/mod.rs rename to crates/icp-app/src/network/managed/mod.rs diff --git a/crates/icp/src/network/managed/run.rs b/crates/icp-app/src/network/managed/run.rs similarity index 97% rename from crates/icp/src/network/managed/run.rs rename to crates/icp-app/src/network/managed/run.rs index 81e5f396f..d12a8a683 100644 --- a/crates/icp/src/network/managed/run.rs +++ b/crates/icp-app/src/network/managed/run.rs @@ -25,20 +25,20 @@ use tracing::{debug, info}; use url::Url; use uuid::Uuid; -use crate::{ - fs::{create_dir_all, lock::LockError, remove_dir_all}, - network::{ - Managed, ManagedLauncherConfig, ManagedMode, NetworkDirectory, Port, - config::{ChildLocator, NetworkDescriptorGatewayPort, NetworkDescriptorModel}, - directory::{ - CheckPortInUseError, PortInUseError, SaveNetworkDescriptorError, - save_network_descriptors, - }, - managed::{ - docker::{DockerDropGuard, ManagedImageOptions, spawn_docker_launcher}, - launcher::{ChildSignalOnDrop, launcher_settings_flags, spawn_network_launcher}, - }, +use crate::network::{ + NetworkDirectory, + config::{ChildLocator, NetworkDescriptorGatewayPort, NetworkDescriptorModel}, + directory::{ + CheckPortInUseError, PortInUseError, SaveNetworkDescriptorError, save_network_descriptors, + }, + managed::{ + docker::{DockerDropGuard, ManagedImageOptions, spawn_docker_launcher}, + launcher::{ChildSignalOnDrop, launcher_settings_flags, spawn_network_launcher}, }, +}; +use icp::{ + fs::{create_dir_all, lock::LockError, remove_dir_all}, + network::{Managed, ManagedLauncherConfig, ManagedMode, Port}, prelude::*, signal::stop_signal, }; @@ -94,7 +94,7 @@ pub async fn stop_network(locator: &ChildLocator) -> Result<(), StopNetworkError #[derive(Debug, Snafu)] pub enum RunNetworkError { #[snafu(transparent)] - CreateDirFailed { source: crate::fs::IoError }, + CreateDirFailed { source: icp::fs::IoError }, #[snafu(transparent)] LockFileError { source: LockError }, @@ -412,13 +412,13 @@ pub enum RunNetworkLauncherError { CreateStatusDir { source: std::io::Error }, #[snafu(display("failed to create dir"))] - CreateDirAll { source: crate::fs::IoError }, + CreateDirAll { source: icp::fs::IoError }, #[snafu(display("failed to remove dir"))] - RemoveDirAll { source: crate::fs::IoError }, + RemoveDirAll { source: icp::fs::IoError }, #[snafu(display("failed to remove file"))] - RemoveFile { source: crate::fs::IoError }, + RemoveFile { source: icp::fs::IoError }, #[snafu(transparent)] SaveNetworkDescriptor { source: SaveNetworkDescriptorError }, @@ -820,7 +820,7 @@ async fn install_proxy( #[cfg(test)] mod tests { use super::*; - use crate::network::{Gateway, ManagedLauncherConfig, Port}; + use icp::network::{Gateway, ManagedLauncherConfig, Port}; #[test] fn transform_native_launcher_default_config() { diff --git a/crates/icp-app/src/network/mod.rs b/crates/icp-app/src/network/mod.rs new file mode 100644 index 000000000..89830a30f --- /dev/null +++ b/crates/icp-app/src/network/mod.rs @@ -0,0 +1,20 @@ +//! Running, describing and reaching networks on this machine. +//! +//! The *configuration* of a network is part of a project, and lives in +//! [`crate::network`]. Everything here is about the machine: launching a managed +//! network, the descriptors it writes, the friendly-domain file its gateway +//! serves, and resolving any network to endpoints and a root key. + +pub mod accessor; +pub mod config; +pub mod custom_domains; +pub mod directory; +pub mod managed; +pub mod resolve; + +#[cfg(any(test, feature = "test-util"))] +pub use accessor::UnimplementedMockDirectories; +pub use accessor::{Accessor, Directories, LocateNetworkDirectoryError}; +pub use directory::{LoadPidError, NetworkDirectory, SavePidError}; +pub use managed::run::{RunNetworkError, run_network}; +pub use resolve::GetNetworkAccessError; diff --git a/crates/icp-app/src/network/resolve.rs b/crates/icp-app/src/network/resolve.rs new file mode 100644 index 000000000..0ad701b64 --- /dev/null +++ b/crates/icp-app/src/network/resolve.rs @@ -0,0 +1,173 @@ +//! Resolving a network to something you can talk to. +//! +//! A managed network is described by a descriptor its launcher wrote, so +//! reaching one means reading that file and checking the port is still the one +//! it claims. A connected network carries its own endpoints, and its root key +//! is either pinned or fetched trust-on-first-use. + +use std::sync::Arc; + +use ic_agent::{AgentError, identity::AnonymousIdentity}; +use snafu::{OptionExt, ResultExt, Snafu}; +use url::Url; + +use icp::network::{Connected, NetworkAccess, NetworkUrls, RootKeySource, RootKeySpec}; +use icp::prelude::*; + +use crate::{ + agent::{Create, CreateAgentError}, + network::{NetworkDirectory, config::NetworkDescriptorModel, directory::LoadNetworkFileError}, +}; + +#[derive(Debug, Snafu)] +pub enum GetNetworkAccessError { + #[snafu(display("failed to load port {port} descriptor"))] + LoadPortDescriptor { + port: u16, + source: LoadNetworkFileError, + }, + + #[snafu(display("the {network} network for this project is not running"))] + NetworkNotRunning { network: String }, + + #[snafu(display( + "port {port} is already in use by the {network} network of another project at {project_dir}" + ))] + NetworkRunningOtherProject { + network: String, + port: u16, + project_dir: PathBuf, + }, + + #[snafu(display("no descriptor found for port {port}"))] + NoPortDescriptor { port: u16 }, + + #[snafu(display("failed to load network descriptor"))] + LoadNetworkDescriptor { source: LoadNetworkFileError }, + + #[snafu(display("failed to create agent to fetch root key from {url}"))] + CreateBootstrapAgent { + url: Url, + #[snafu(source(from(CreateAgentError, Box::new)))] + source: Box, + }, + + #[snafu(display("failed to fetch root key from {url}"))] + FetchRootKey { + url: Url, + #[snafu(source(from(AgentError, Box::new)))] + source: Box, + }, +} + +pub async fn get_managed_network_access( + nd: NetworkDirectory, +) -> Result { + let (desc, gateway_url) = managed_network_gateway(nd).await?; + Ok(NetworkAccess { + root_key: desc.root_key, + root_key_source: RootKeySource::Managed, + api_url: gateway_url.clone(), + http_gateway_url: Some(gateway_url), + use_friendly_domains: desc.use_friendly_domains, + }) +} + +/// The URLs a running managed network is reached at. Its gateway serves the API +/// as well, so both URLs are the same one. +pub async fn get_managed_network_urls( + nd: NetworkDirectory, +) -> Result { + let (_, gateway_url) = managed_network_gateway(nd).await?; + Ok(NetworkUrls { + api_url: gateway_url.clone(), + http_gateway_url: Some(gateway_url), + }) +} + +/// A running managed network's descriptor and the URL its gateway is reachable +/// at. A network that is not running has no descriptor, and one whose fixed port +/// has since been taken by another project's network is not the network the +/// descriptor describes — both are errors rather than a URL nothing answers on. +async fn managed_network_gateway( + nd: NetworkDirectory, +) -> Result<(NetworkDescriptorModel, Url), GetNetworkAccessError> { + // Load network descriptor + let desc = nd + .load_network_descriptor() + .await + .context(LoadNetworkDescriptorSnafu)? + .ok_or(GetNetworkAccessError::NetworkNotRunning { + network: nd.network_name.to_owned(), + })?; + + // Specify port + let port = desc.gateway.port; + + // Apply gateway configuration + if desc.gateway.fixed { + let pdesc = nd + .load_port_descriptor(port) + .await + .context(LoadPortDescriptorSnafu { port })? + .context(NoPortDescriptorSnafu { port })?; + + if desc.id != pdesc.id { + return NetworkRunningOtherProjectSnafu { + network: pdesc.network, + port: pdesc.gateway.port, + project_dir: pdesc.project_dir, + } + .fail(); + } + } + let http_gateway_url = Url::parse(&format!("http://{}:{port}", desc.gateway.host)).unwrap(); + Ok((desc, http_gateway_url)) +} + +pub async fn get_connected_network_access( + connected: &Connected, + agent: &Arc, +) -> Result { + let (root_key, root_key_source) = match &connected.root_key { + RootKeySpec::Mainnet => (IC_ROOT_KEY.to_vec(), RootKeySource::Mainnet), + RootKeySpec::Explicit(bytes) => (bytes.clone(), RootKeySource::Configured), + RootKeySpec::Fetch => { + let root_key = fetch_root_key(agent, &connected.api_url).await?; + (root_key, RootKeySource::Fetched) + } + }; + + Ok(NetworkAccess { + root_key, + root_key_source, + api_url: connected.api_url.clone(), + http_gateway_url: connected.http_gateway_url.clone(), + use_friendly_domains: false, + }) +} + +/// Fetch a network's root key trust-on-first-use. This does *not* verify the +/// key's provenance, so we warn the user that responses cannot be trusted the +/// way a pinned key allows. +async fn fetch_root_key( + agent: &Arc, + api_url: &Url, +) -> Result, GetNetworkAccessError> { + tracing::warn!( + "fetching the root key from {api_url}; its provenance is not verified (trust-on-first-use)" + ); + let bootstrap = agent + .create(Arc::new(AnonymousIdentity), api_url.as_str(), None) + .await + .context(CreateBootstrapAgentSnafu { + url: api_url.clone(), + })?; + bootstrap + .fetch_root_key() + .await + .context(FetchRootKeySnafu { + url: api_url.clone(), + })?; + Ok(bootstrap.read_root_key()) +} diff --git a/crates/icp/src/operations/canister_migration.rs b/crates/icp-app/src/operations/canister_migration.rs similarity index 100% rename from crates/icp/src/operations/canister_migration.rs rename to crates/icp-app/src/operations/canister_migration.rs diff --git a/crates/icp-app/src/operations/mod.rs b/crates/icp-app/src/operations/mod.rs new file mode 100644 index 000000000..f4e191327 --- /dev/null +++ b/crates/icp-app/src/operations/mod.rs @@ -0,0 +1,6 @@ +//! Operations that act on a canister the caller named by principal, or on a +//! ledger, rather than on something a project manifest describes. + +pub mod canister_migration; +pub mod snapshot_transfer; +pub mod token; diff --git a/crates/icp/src/operations/snapshot_transfer.rs b/crates/icp-app/src/operations/snapshot_transfer.rs similarity index 96% rename from crates/icp/src/operations/snapshot_transfer.rs rename to crates/icp-app/src/operations/snapshot_transfer.rs index a6de62176..686c6036b 100644 --- a/crates/icp/src/operations/snapshot_transfer.rs +++ b/crates/icp-app/src/operations/snapshot_transfer.rs @@ -13,10 +13,10 @@ use ic_management_canister_types::{ UploadCanisterSnapshotMetadataResult, }; -use super::proxy::UpdateOrProxyError; -use super::proxy_management; -use crate::operations::task::TaskReporter; -use crate::{ +use icp::operations::proxy::UpdateOrProxyError; +use icp::operations::proxy_management; +use icp::operations::task::TaskReporter; +use icp::{ fs::lock::{DirectoryStructureLock, LWrite, LockError, PathsAccess}, prelude::*, }; @@ -79,9 +79,9 @@ impl SnapshotPaths { } /// Ensure the directory and wasm chunk store subdirectory exist. - pub fn ensure_dirs(&self) -> Result<(), crate::fs::IoError> { - crate::fs::create_dir_all(&self.dir)?; - crate::fs::create_dir_all(&self.wasm_chunk_store_dir())?; + pub fn ensure_dirs(&self) -> Result<(), icp::fs::IoError> { + icp::fs::create_dir_all(&self.dir)?; + icp::fs::create_dir_all(&self.wasm_chunk_store_dir())?; Ok(()) } @@ -147,13 +147,13 @@ pub enum SnapshotTransferError { }, #[snafu(transparent)] - FsIo { source: crate::fs::IoError }, + FsIo { source: icp::fs::IoError }, #[snafu(transparent)] - FsRename { source: crate::fs::RenameError }, + FsRename { source: icp::fs::RenameError }, #[snafu(transparent)] - Json { source: crate::fs::json::Error }, + Json { source: icp::fs::json::Error }, #[snafu(transparent)] Lock { source: LockError }, @@ -505,7 +505,7 @@ pub async fn download_blob_to_file( let output_path = paths.blob_path(blob_type); if total_size == 0 { - crate::fs::write(&output_path, &[])?; + icp::fs::write(&output_path, &[])?; return Ok(()); } @@ -632,7 +632,7 @@ pub async fn download_wasm_chunk( .await .context(ReadWasmChunkSnafu { hash: &hash_hex })?; - crate::fs::write(&output_path, &result.chunk)?; + icp::fs::write(&output_path, &result.chunk)?; Ok(()) } @@ -769,7 +769,7 @@ pub async fn upload_wasm_chunk( paths: LWrite<&SnapshotPaths>, ) -> Result<(), SnapshotTransferError> { let chunk_path = paths.wasm_chunk_path(chunk_hash); - let chunk = crate::fs::read(&chunk_path)?; + let chunk = icp::fs::read(&chunk_path)?; let args = UploadCanisterSnapshotDataArgs { canister_id, @@ -795,7 +795,7 @@ pub fn save_upload_progress( progress: &UploadProgress, paths: LWrite<&SnapshotPaths>, ) -> Result<(), SnapshotTransferError> { - crate::fs::json::save(&paths.upload_progress_path(), progress)?; + icp::fs::json::save(&paths.upload_progress_path(), progress)?; Ok(()) } @@ -809,14 +809,14 @@ pub fn load_upload_progress( path: paths.dir().to_path_buf(), }); } - Ok(crate::fs::json::load(&progress_path)?) + Ok(icp::fs::json::load(&progress_path)?) } /// Delete upload progress file. pub fn delete_upload_progress(paths: LWrite<&SnapshotPaths>) -> Result<(), SnapshotTransferError> { let progress_path = paths.upload_progress_path(); if progress_path.exists() { - crate::fs::remove_file(&progress_path)?; + icp::fs::remove_file(&progress_path)?; } Ok(()) } @@ -843,7 +843,7 @@ pub fn save_download_progress( drop(file); // Atomic rename - crate::fs::rename(&tmp_path, &target_path)?; + icp::fs::rename(&tmp_path, &target_path)?; Ok(()) } @@ -852,7 +852,7 @@ pub fn save_download_progress( pub fn load_download_progress( paths: LWrite<&SnapshotPaths>, ) -> Result { - Ok(crate::fs::json::load_or_default( + Ok(icp::fs::json::load_or_default( &paths.download_progress_path(), )?) } @@ -863,7 +863,7 @@ pub fn delete_download_progress( ) -> Result<(), SnapshotTransferError> { let progress_path = paths.download_progress_path(); if progress_path.exists() { - crate::fs::remove_file(&progress_path)?; + icp::fs::remove_file(&progress_path)?; } Ok(()) } @@ -873,7 +873,7 @@ pub fn save_metadata( metadata: &ReadCanisterSnapshotMetadataResult, paths: LWrite<&SnapshotPaths>, ) -> Result<(), SnapshotTransferError> { - crate::fs::json::save(&paths.metadata_path(), metadata)?; + icp::fs::json::save(&paths.metadata_path(), metadata)?; Ok(()) } @@ -887,5 +887,5 @@ pub fn load_metadata( path: metadata_path, }); } - Ok(crate::fs::json::load(&metadata_path)?) + Ok(icp::fs::json::load(&metadata_path)?) } diff --git a/crates/icp/src/operations/token/allowance.rs b/crates/icp-app/src/operations/token/allowance.rs similarity index 100% rename from crates/icp/src/operations/token/allowance.rs rename to crates/icp-app/src/operations/token/allowance.rs diff --git a/crates/icp/src/operations/token/approve.rs b/crates/icp-app/src/operations/token/approve.rs similarity index 99% rename from crates/icp/src/operations/token/approve.rs rename to crates/icp-app/src/operations/token/approve.rs index 9a5ca4723..529a9e748 100644 --- a/crates/icp/src/operations/token/approve.rs +++ b/crates/icp-app/src/operations/token/approve.rs @@ -1,7 +1,7 @@ -use crate::parsers::to_token_unit_amount; use bigdecimal::BigDecimal; use candid::{Decode, Encode, Nat, Principal}; use ic_agent::{Agent, AgentError}; +use icp::parsers::to_token_unit_amount; use icrc_ledger_types::icrc1::account::Account; use icrc_ledger_types::icrc2::approve::{ApproveArgs, ApproveError as Icrc2ApproveError}; use snafu::{ResultExt, Snafu}; diff --git a/crates/icp/src/operations/token/balance.rs b/crates/icp-app/src/operations/token/balance.rs similarity index 100% rename from crates/icp/src/operations/token/balance.rs rename to crates/icp-app/src/operations/token/balance.rs diff --git a/crates/icp/src/operations/token/mint.rs b/crates/icp-app/src/operations/token/mint.rs similarity index 100% rename from crates/icp/src/operations/token/mint.rs rename to crates/icp-app/src/operations/token/mint.rs diff --git a/crates/icp/src/operations/token/mod.rs b/crates/icp-app/src/operations/token/mod.rs similarity index 100% rename from crates/icp/src/operations/token/mod.rs rename to crates/icp-app/src/operations/token/mod.rs diff --git a/crates/icp/src/operations/token/transfer.rs b/crates/icp-app/src/operations/token/transfer.rs similarity index 99% rename from crates/icp/src/operations/token/transfer.rs rename to crates/icp-app/src/operations/token/transfer.rs index 9123590cf..632d01307 100644 --- a/crates/icp/src/operations/token/transfer.rs +++ b/crates/icp-app/src/operations/token/transfer.rs @@ -13,7 +13,7 @@ use icrc_ledger_types::icrc1::{ use num_traits::ToPrimitive; use snafu::{ResultExt, Snafu}; -use crate::parsers::FlexibleAccountId; +use icp::parsers::FlexibleAccountId; use super::{TOKEN_LEDGER_CIDS, TokenAmount}; diff --git a/crates/icp/src/package.rs b/crates/icp-app/src/package.rs similarity index 80% rename from crates/icp/src/package.rs rename to crates/icp-app/src/package.rs index 4b62e195c..abc28a2b1 100644 --- a/crates/icp/src/package.rs +++ b/crates/icp-app/src/package.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use snafu::prelude::*; -use crate::{ +use icp::{ fs::lock::{DirectoryStructureLock, LRead, LWrite, LockError, PathsAccess}, prelude::*, }; @@ -82,13 +82,13 @@ pub fn cache_wasm( cache: LWrite<&PackageCachePaths>, sha: &str, wasm: &[u8], -) -> Result<(), crate::fs::IoError> { +) -> Result<(), icp::fs::IoError> { let cache_path = cache.wasm_sha(sha); let cache_wasm_path = cache_path.wasm(); if !cache_wasm_path.exists() { - crate::fs::create_dir_all(cache_path.dir())?; - crate::fs::write(&cache_wasm_path, wasm)?; - _ = crate::fs::write(&cache_path.atime(), b""); + icp::fs::create_dir_all(cache_path.dir())?; + icp::fs::write(&cache_wasm_path, wasm)?; + _ = icp::fs::write(&cache_path.atime(), b""); } Ok(()) } @@ -138,8 +138,8 @@ pub fn read_cached_recipe( let cache_path = cache.recipe_sha(cache_key); let template_path = cache_path.template(); if template_path.exists() { - let template = crate::fs::read(&template_path).context(RecipeCacheIoSnafu)?; - _ = crate::fs::write(&cache_path.atime(), b""); + let template = icp::fs::read(&template_path).context(RecipeCacheIoSnafu)?; + _ = icp::fs::write(&cache_path.atime(), b""); Ok(Some(template)) } else { Ok(None) @@ -183,9 +183,9 @@ pub fn cache_recipe( let cache_path = cache.recipe_sha(cache_key); let template_path = cache_path.template(); if !template_path.exists() { - crate::fs::create_dir_all(cache_path.dir()).context(RecipeCacheIoSnafu)?; - crate::fs::write(&template_path, template).context(RecipeCacheIoSnafu)?; - _ = crate::fs::write(&cache_path.atime(), b""); + icp::fs::create_dir_all(cache_path.dir()).context(RecipeCacheIoSnafu)?; + icp::fs::write(&template_path, template).context(RecipeCacheIoSnafu)?; + _ = icp::fs::write(&cache_path.atime(), b""); } Ok(()) } @@ -193,21 +193,23 @@ pub fn cache_recipe( #[derive(Debug, Snafu)] pub enum RecipeCacheError { #[snafu(display("failed to load recipe cache tag"))] - LoadRecipeTag { source: crate::fs::json::Error }, + LoadRecipeTag { source: icp::fs::json::Error }, #[snafu(display("failed to save recipe cache tag"))] - SaveRecipeTag { source: crate::fs::json::Error }, + SaveRecipeTag { source: icp::fs::json::Error }, #[snafu(display("failed to read or write recipe cache file"))] - RecipeCacheIo { source: crate::fs::IoError }, + RecipeCacheIo { source: icp::fs::IoError }, } pub type PackageCache = DirectoryStructureLock; -impl PackageCache { - pub fn new(root: PathBuf) -> Result { - DirectoryStructureLock::open_or_create(PackageCachePaths { root }) - } +/// Opens (creating if needed) the package cache rooted at `root`. +/// +/// A free function rather than an inherent one: [`PackageCache`] is an alias +/// for a lock type that belongs to another crate. +pub fn open(root: PathBuf) -> Result { + DirectoryStructureLock::open_or_create(PackageCachePaths { root }) } impl PathsAccess for PackageCachePaths { @@ -220,8 +222,8 @@ pub fn get_tag( paths: LRead<&PackageCachePaths>, tool: &str, tag: &str, -) -> Result, crate::fs::json::Error> { - let manifest: Manifest = crate::fs::json::load_or_default(&paths.manifest())?; +) -> Result, icp::fs::json::Error> { + let manifest: Manifest = icp::fs::json::load_or_default(&paths.manifest())?; Ok(manifest.tags.get(&format!("{tool}:{tag}")).cloned()) } @@ -230,12 +232,12 @@ pub fn set_tag( tool: &str, version: &str, tag: &str, -) -> Result<(), crate::fs::json::Error> { - let mut manifest: Manifest = crate::fs::json::load_or_default(&paths.manifest())?; +) -> Result<(), icp::fs::json::Error> { + let mut manifest: Manifest = icp::fs::json::load_or_default(&paths.manifest())?; manifest .tags .insert(format!("{tool}:{tag}"), version.to_string()); - crate::fs::json::save(&paths.manifest(), &manifest)?; + icp::fs::json::save(&paths.manifest(), &manifest)?; Ok(()) } @@ -246,15 +248,15 @@ pub fn set_tag_with_updater( version: &str, tag: &str, updater_version: &str, -) -> Result<(), crate::fs::json::Error> { - let mut manifest: Manifest = crate::fs::json::load_or_default(&paths.manifest())?; +) -> Result<(), icp::fs::json::Error> { + let mut manifest: Manifest = icp::fs::json::load_or_default(&paths.manifest())?; manifest .tags .insert(format!("{tool}:{tag}"), version.to_string()); manifest .updater_versions .insert(tool.to_string(), updater_version.to_string()); - crate::fs::json::save(&paths.manifest(), &manifest)?; + icp::fs::json::save(&paths.manifest(), &manifest)?; Ok(()) } @@ -264,8 +266,8 @@ pub fn get_tag_with_updater( paths: LRead<&PackageCachePaths>, tool: &str, tag: &str, -) -> Result<(Option, Option), crate::fs::json::Error> { - let manifest: Manifest = crate::fs::json::load_or_default(&paths.manifest())?; +) -> Result<(Option, Option), icp::fs::json::Error> { + let manifest: Manifest = icp::fs::json::load_or_default(&paths.manifest())?; let tag_value = manifest.tags.get(&format!("{tool}:{tag}")).cloned(); let updater = manifest.updater_versions.get(tool).cloned(); Ok((tag_value, updater)) diff --git a/crates/icp/src/canister/recipe/fetch.rs b/crates/icp-app/src/recipe.rs similarity index 87% rename from crates/icp/src/canister/recipe/fetch.rs rename to crates/icp-app/src/recipe.rs index cb6ca5c2d..d6989f7d2 100644 --- a/crates/icp/src/canister/recipe/fetch.rs +++ b/crates/icp-app/src/recipe.rs @@ -9,17 +9,17 @@ use snafu::prelude::*; use tracing::debug; use url::ParseError; -use crate::{ +use crate::package::{ + PackageCache, cache_registry_recipe, cache_uri_recipe, read_cached_registry_recipe, + read_cached_uri_recipe, +}; +use icp::{ fs::read, manifest::recipe::{Recipe, RecipeType}, - package::{ - PackageCache, cache_registry_recipe, cache_uri_recipe, read_cached_registry_recipe, - read_cached_uri_recipe, - }, prelude::*, }; -use super::{CommitSnafu, FetchSnafu, Resolve, ResolveError}; +use icp::canister::recipe::{Fetched, Resolve, ResolveError}; /// Fetches recipe templates over HTTP, caching downloads in the package cache. /// Template *rendering* is a separate stage @@ -29,21 +29,20 @@ pub struct RecipeFetcher { /// Http client for fetching remote recipe templates pub http_client: reqwest::Client, /// Package cache for caching downloaded recipe templates - pub pkg_cache: PackageCache, + pub pkg_cache: std::sync::Arc, } -/// The result of the fetch stage. -pub struct Fetched { +/// The result of the fetch stage, as this resolver sees it. +struct FetchedTemplate { /// Raw Handlebars template source. - pub template: String, + template: String, /// A cache write deliberately held back until the template is known to /// render; `None` when there is nothing to cache (a local file or a cache /// hit) or when the download was already cached because it was checksummed. /// - /// Pass to [`Resolve::commit`] after [`render_recipe`](super::render_recipe) - /// succeeds. - pub pending_cache: Option, + /// Committed by [`Resolve::commit`] once the template has rendered. + pending_cache: Option, } /// A cache write for an unpinned download, held until the template renders. @@ -77,7 +76,7 @@ enum TemplateSource { #[derive(Debug, Snafu)] pub enum RecipeFetchError { #[snafu(display("failed to read local recipe template file"))] - ReadFile { source: crate::fs::IoError }, + ReadFile { source: icp::fs::IoError }, #[snafu(display("failed to decode UTF-8 string"))] DecodeUtf8 { source: FromUtf8Error }, @@ -107,7 +106,7 @@ pub enum RecipeFetchError { }, #[snafu(display("failed to acquire lock on package cache"))] - LockCache { source: crate::fs::lock::LockError }, + LockCache { source: icp::fs::lock::LockError }, } impl RecipeFetcher { @@ -117,7 +116,7 @@ impl RecipeFetcher { /// A checksummed download is cached here. An unpinned one is returned as a /// [`PendingCache`] for the caller to commit once it renders — see /// [`PendingCache`] for why. - async fn fetch_recipe(&self, recipe: &Recipe) -> Result { + async fn fetch_recipe(&self, recipe: &Recipe) -> Result { // Determine the template source let tmpl_source = match &recipe.recipe_type { RecipeType::File(path) => TemplateSource::LocalPath(Path::new(&path).into()), @@ -196,7 +195,7 @@ impl RecipeFetcher { // Nothing was downloaded (local file, or a cache hit): nothing to cache. if !should_cache { - return Ok(Fetched { + return Ok(FetchedTemplate { template: tmpl, pending_cache: None, }); @@ -221,18 +220,35 @@ impl RecipeFetcher { // so cache it now. An unpinned one waits for a successful render. if recipe.sha256.is_some() { self.write_cache(&pending).await?; - return Ok(Fetched { + return Ok(FetchedTemplate { template: pending.template, pending_cache: None, }); } - Ok(Fetched { + Ok(FetchedTemplate { template: pending.template.clone(), pending_cache: Some(pending), }) } + /// Where a recipe's template belongs in the cache, or `None` for a recipe + /// that is never cached because it is read from the project itself. + fn cache_target(&self, recipe: &Recipe) -> Option { + match &recipe.recipe_type { + RecipeType::File(_) => None, + RecipeType::Url(u) => Some(CacheTarget::Uri(u.clone())), + RecipeType::Registry { + name, + recipe: recipe_name, + version, + } => Some(CacheTarget::Registry { + package: format!("@{name}/{recipe_name}"), + version: version.clone(), + }), + } + } + /// Write a fetched template into the package cache. async fn write_cache(&self, pending: &PendingCache) -> Result<(), RecipeFetchError> { let hash = hex::encode(pending.hash); @@ -286,11 +302,27 @@ impl RecipeFetcher { #[async_trait] impl Resolve for RecipeFetcher { async fn resolve(&self, recipe: &Recipe) -> Result { - self.fetch_recipe(recipe).await.context(FetchSnafu) + let fetched = self.fetch_recipe(recipe).await.map_err(ResolveError::new)?; + Ok(Fetched { + template: fetched.template, + deferred: fetched.pending_cache.is_some(), + }) } - async fn commit(&self, pending: PendingCache) -> Result<(), ResolveError> { - self.write_cache(&pending).await.context(CommitSnafu) + /// The write `resolve` held back is rebuilt here from the recipe and the + /// template that rendered, rather than carried across the trait: where a + /// template belongs in the cache is this resolver's business, and the + /// project layer has no use for a value it can only hand straight back. + async fn commit(&self, recipe: &Recipe, fetched: &Fetched) -> Result<(), ResolveError> { + let Some(target) = self.cache_target(recipe) else { + return Ok(()); + }; + let pending = PendingCache { + target, + hash: Sha256::digest(fetched.template.as_bytes()).into(), + template: fetched.template.clone(), + }; + self.write_cache(&pending).await.map_err(ResolveError::new) } } @@ -320,12 +352,12 @@ fn parse_bytes_to_string(bytes: Vec) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::manifest::recipe::{Recipe, RecipeType}; + use icp::manifest::recipe::{Recipe, RecipeType}; fn fetcher(cache_dir: &Path) -> RecipeFetcher { RecipeFetcher { http_client: reqwest::Client::new(), - pkg_cache: PackageCache::new(cache_dir.to_owned()).unwrap(), + pkg_cache: std::sync::Arc::new(crate::package::open(cache_dir.to_owned()).unwrap()), } } @@ -446,11 +478,11 @@ mod tests { ); // Rendering fails, so the caller never commits. - let ctx = super::super::RecipeContext { + let ctx = icp::canister::recipe::RecipeContext { canister_name: "c".to_owned(), }; assert!( - super::super::render_recipe(&fetched.template, &recipe, &ctx).is_err(), + icp::canister::recipe::render_recipe(&fetched.template, &recipe, &ctx).is_err(), "fixture template must fail to render" ); diff --git a/crates/icp/src/settings.rs b/crates/icp-app/src/settings.rs similarity index 95% rename from crates/icp/src/settings.rs rename to crates/icp-app/src/settings.rs index 8d38de949..adcbb635c 100644 --- a/crates/icp/src/settings.rs +++ b/crates/icp-app/src/settings.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use snafu::{Snafu, ensure}; -use crate::{ +use icp::{ fs::{ json, lock::{DirectoryStructureLock, LRead, LWrite, LockError, PathsAccess}, @@ -32,8 +32,8 @@ impl SettingsPaths { } /// Ensures the settings directory exists and returns the path to the settings file. - pub fn ensure_settings_path(&self) -> Result { - crate::fs::create_dir_all(&self.dir)?; + pub fn ensure_settings_path(&self) -> Result { + icp::fs::create_dir_all(&self.dir)?; Ok(self.settings_path()) } } @@ -137,7 +137,7 @@ pub enum WriteSettingsError { WriteJsonError { source: json::Error }, #[snafu(transparent)] - CreateDirectoryError { source: crate::fs::IoError }, + CreateDirectoryError { source: icp::fs::IoError }, } #[derive(Debug, Snafu)] diff --git a/crates/icp/src/signed_message.rs b/crates/icp-app/src/signed_message.rs similarity index 99% rename from crates/icp/src/signed_message.rs rename to crates/icp-app/src/signed_message.rs index d53eeaf9c..f4971b0b5 100644 --- a/crates/icp/src/signed_message.rs +++ b/crates/icp-app/src/signed_message.rs @@ -16,8 +16,6 @@ //! 3. **Display only** — `candid` and `summary`. Never used for a decision; //! everything shown to the operator is re-derived from the envelope. -use crate::network::RootKeySpec; -use crate::prelude::*; use base64::engine::general_purpose::STANDARD as BASE64; use candid::Principal; use ic_agent::agent::{ @@ -25,6 +23,8 @@ use ic_agent::agent::{ signed_update_inspect, }; use ic_agent::{AgentError, RequestId}; +use icp::network::RootKeySpec; +use icp::prelude::*; use serde::{Deserialize, Serialize}; use snafu::prelude::*; use time::{Duration, OffsetDateTime, UtcOffset, format_description::well_known::Rfc3339}; @@ -230,7 +230,7 @@ pub struct Validated { impl SignedMessage { /// Writes the message to `path`. pub fn save(&self, path: &Path) -> Result<(), Error> { - crate::fs::json::save(path, self).context(SaveSnafu { path }) + icp::fs::json::save(path, self).context(SaveSnafu { path }) } /// Renders the message exactly as [`SignedMessage::save`] would write it, for @@ -242,7 +242,7 @@ impl SignedMessage { /// Reads a message from `path`. The result is unvalidated — call /// [`SignedMessage::validate`] before acting on any of it. pub fn load(path: &Path) -> Result { - crate::fs::json::load(path).context(LoadSnafu { path }) + icp::fs::json::load(path).context(LoadSnafu { path }) } /// Checks the file against its envelope and reports where `now` falls in the @@ -481,7 +481,7 @@ pub fn format_timestamp(t: OffsetDateTime) -> String { pub enum Error { #[snafu(display("failed to write the signed message to {path}"))] Save { - source: crate::fs::json::Error, + source: icp::fs::json::Error, path: PathBuf, }, @@ -490,7 +490,7 @@ pub enum Error { #[snafu(display("failed to read the signed message at {path}"))] Load { - source: crate::fs::json::Error, + source: icp::fs::json::Error, path: PathBuf, }, diff --git a/crates/icp/src/telemetry_data.rs b/crates/icp-app/src/telemetry_data.rs similarity index 82% rename from crates/icp/src/telemetry_data.rs rename to crates/icp-app/src/telemetry_data.rs index d74a96783..3f13ec548 100644 --- a/crates/icp/src/telemetry_data.rs +++ b/crates/icp-app/src/telemetry_data.rs @@ -44,7 +44,7 @@ impl TelemetryData { *self.network_type.lock().unwrap() } - pub fn set_project(&self, project: &crate::Project) { + fn set_project(&self, project: &icp::Project) { let recipes: Vec = project .canisters .values() @@ -63,6 +63,20 @@ impl TelemetryData { } } +/// The project facts telemetry keeps are established during environment +/// resolution, so the bag receives them from there rather than the other way +/// around. +impl icp::host::Observe for TelemetryData { + fn environment_resolved(&self, project: &icp::Project, environment: &icp::Environment) { + let network_type = match &environment.network.configuration { + icp::network::Configuration::Managed { .. } => NetworkType::Managed, + icp::network::Configuration::Connected { .. } => NetworkType::Connected, + }; + self.set_network_type(network_type); + self.set_project(project); + } +} + /// What form of authentication mechanism an identity uses. #[derive(Clone, Copy, Debug, Serialize, serde::Deserialize)] #[serde(rename_all = "kebab-case")] diff --git a/crates/icp-app/src/wasm.rs b/crates/icp-app/src/wasm.rs new file mode 100644 index 000000000..be298e27a --- /dev/null +++ b/crates/icp-app/src/wasm.rs @@ -0,0 +1,176 @@ +//! Getting hold of a wasm module a manifest names but the project does not +//! contain: over HTTP when it is a URL, from the package cache when it has +//! been fetched before. + +use std::sync::Arc; + +use camino::{Utf8Path, Utf8PathBuf}; +use icp::canister::wasm::{Fetch, FetchError}; +use icp::fs::read; +use icp::manifest::prebuilt::SourceField; +use icp::prelude::*; +use icp_events::StepReporter; +use reqwest::{Client, Method, Request}; +use sha2::{Digest, Sha256}; +use snafu::prelude::*; +use url::Url; + +use crate::package::{PackageCache, cache_wasm}; + +#[derive(Debug, Snafu)] +pub enum WasmError { + #[snafu(display("failed to read wasm file at '{path}'"))] + ReadLocal { + source: icp::fs::IoError, + path: Utf8PathBuf, + }, + + #[snafu(display("failed to parse wasm url"))] + ParseUrl { source: url::ParseError }, + + #[snafu(display("failed to fetch wasm file"))] + HttpRequest { source: reqwest::Error }, + + #[snafu(display("http request failed: {status}"))] + HttpStatus { status: reqwest::StatusCode }, + + #[snafu(display("failed to read http response"))] + HttpResponse { source: reqwest::Error }, + + #[snafu(display("checksum mismatch, expected: {expected}, actual: {actual}"))] + ChecksumMismatch { expected: String, actual: String }, + + #[snafu(display("failed to cache wasm file"))] + CacheFile { source: icp::fs::IoError }, + + #[snafu(display("failed to acquire lock on package cache"))] + LockCache { source: icp::fs::lock::LockError }, +} + +/// The [`Fetch`] that downloads over HTTP and caches in the package cache. +pub struct Fetcher { + http_client: Client, + pkg_cache: Arc, +} + +impl Fetcher { + pub fn new(http_client: Client, pkg_cache: Arc) -> Self { + Self { + http_client, + pkg_cache, + } + } +} + +#[async_trait::async_trait] +impl Fetch for Fetcher { + /// - Local: verifies sha256 if provided, returns the local path. + /// - Remote with sha256: checks the cache first; downloads, verifies, and caches on miss. + /// - Remote without sha256: always downloads, computes sha256, caches by the computed sha256. + async fn wasm( + &self, + source: &SourceField, + base_dir: &Utf8Path, + sha256: Option<&str>, + reporter: &StepReporter, + ) -> Result { + self.fetch(source, base_dir, sha256, reporter) + .await + .map_err(FetchError::new) + } +} + +impl Fetcher { + async fn fetch( + &self, + source: &SourceField, + base_dir: &Utf8Path, + sha256: Option<&str>, + reporter: &StepReporter, + ) -> Result { + let pkg_cache = &self.pkg_cache; + match source { + SourceField::Local(s) => { + let path = base_dir.join(&s.path); + if let Some(expected) = sha256 { + reporter.info(format!("Reading wasm: {}", s.path)); + let bytes = read(&path).context(ReadLocalSnafu { + path: s.path.clone(), + })?; + reporter.info("Verifying checksum"); + let actual = hex::encode(Sha256::digest(&bytes)); + ensure!( + actual == expected, + ChecksumMismatchSnafu { + expected: expected.to_owned(), + actual, + } + ); + } + Ok(path) + } + SourceField::Remote(s) => { + // Pre-download cache check is only possible when sha256 is known. + if let Some(expected) = sha256 { + let cached = pkg_cache + .with_read(async |r| { + let wasm_cache = r.wasm_sha(expected); + let path = wasm_cache.wasm(); + if path.exists() { + _ = icp::fs::write(&wasm_cache.atime(), b""); + Some(path) + } else { + None + } + }) + .await + .context(LockCacheSnafu)?; + if let Some(path) = cached { + reporter.info("Using cached file"); + return Ok(path); + } + } + + let url = Url::parse(&s.url).context(ParseUrlSnafu)?; + reporter.info(format!("Fetching wasm: {url}")); + let resp = self + .http_client + .execute(Request::new(Method::GET, url)) + .await + .context(HttpRequestSnafu)?; + let status = resp.status(); + if !status.is_success() { + return HttpStatusSnafu { status }.fail(); + } + let bytes = resp.bytes().await.context(HttpResponseSnafu)?.to_vec(); + + // Use provided sha256 as cache key (after verifying), or compute from bytes. + let cache_sha = match sha256 { + Some(expected) => { + reporter.info("Verifying checksum"); + let actual = hex::encode(Sha256::digest(&bytes)); + ensure!( + actual == expected, + ChecksumMismatchSnafu { + expected: expected.to_owned(), + actual, + } + ); + actual + } + None => hex::encode(Sha256::digest(&bytes)), + }; + + pkg_cache + .with_write(async |w| cache_wasm(w, &cache_sha, &bytes).context(CacheFileSnafu)) + .await + .context(LockCacheSnafu)??; + + pkg_cache + .with_read(async |r| r.wasm_sha(&cache_sha).wasm()) + .await + .context(LockCacheSnafu) + } + } + } +} diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index 9d0427b22..9cec04b5d 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -40,6 +40,7 @@ ic-management-canister-types.workspace = true ic-utils.workspace = true icp-canister-interfaces.workspace = true icp = { workspace = true, features = ["clap"] } +icp-app = { workspace = true, features = ["clap"] } icp-events.workspace = true icrc-ledger-types.workspace = true indicatif.workspace = true diff --git a/crates/icp-cli/src/commands/args.rs b/crates/icp-cli/src/commands/args.rs index 5d71c81b0..9a2c5a7dd 100644 --- a/crates/icp-cli/src/commands/args.rs +++ b/crates/icp-cli/src/commands/args.rs @@ -4,14 +4,12 @@ use anyhow::{Context as _, bail}; use candid::Principal; use clap::{Args, ValueHint}; use clap_complete::ArgValueCandidates; -use icp::identity::IdentitySelection; +use icp::host::{CanisterSelection, EnvironmentSelection}; use icp::manifest::ArgsFormat; use icp::prelude::PathBuf; use icp::{CanisterArgs, fs}; -use icp::{ - context::NetworkSelection, - host::{CanisterSelection, EnvironmentSelection}, -}; +use icp_app::context::NetworkSelection; +use icp_app::identity::IdentitySelection; use crate::options::{EnvironmentOpt, IdentityOpt, NetworkOpt}; diff --git a/crates/icp-cli/src/commands/build.rs b/crates/icp-cli/src/commands/build.rs index de8a8e44e..b4baffdd5 100644 --- a/crates/icp-cli/src/commands/build.rs +++ b/crates/icp-cli/src/commands/build.rs @@ -1,7 +1,8 @@ use clap::Args; use clap_complete::ArgValueCandidates; use futures::future::try_join_all; -use icp::{context::Context, host::EnvironmentSelection}; +use icp::host::EnvironmentSelection; +use icp_app::context::Context; use tracing::info; @@ -58,14 +59,12 @@ pub(crate) async fn exec(ctx: &Context, args: &BuildArgs) -> Result<(), anyhow:: // Build the selected canisters info!("Building canisters:"); - let pkg_cache = ctx.dirs.package_cache()?; rendered(ctx.debug, async |reporter| { build_many( canisters_to_build, environment_selection.name(), ctx.host.builder.clone(), ctx.host.artifacts.clone(), - &pkg_cache, reporter, ) .await diff --git a/crates/icp-cli/src/commands/canister/call.rs b/crates/icp-cli/src/commands/canister/call.rs index 829c4f30d..9c52f086c 100644 --- a/crates/icp-cli/src/commands/canister/call.rs +++ b/crates/icp-cli/src/commands/canister/call.rs @@ -4,17 +4,15 @@ use candid_parser::assist; use candid_parser::parse_idl_args; use clap::{Args, ValueHint}; use ic_agent::agent::EffectiveId; +use icp::host::EnvironmentSelection; use icp::manifest::ArgsFormat; use icp::network::{Configuration as NetworkConfiguration, RootKeySpec}; use icp::parsers::{CyclesAmount, DurationAmount}; use icp::prelude::*; -use icp::signed_message::{ +use icp_app::context::{Context, NetworkSelection}; +use icp_app::signed_message::{ self, CallType, Destination, Request, SignedMessage, Summary, WindowState, }; -use icp::{ - context::{Context, NetworkSelection}, - host::EnvironmentSelection, -}; use std::io::{self, Write}; use std::str::FromStr; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; diff --git a/crates/icp-cli/src/commands/canister/create.rs b/crates/icp-cli/src/commands/canister/create.rs index 3ecea0749..0d2d8a5d0 100644 --- a/crates/icp-cli/src/commands/canister/create.rs +++ b/crates/icp-cli/src/commands/canister/create.rs @@ -6,14 +6,12 @@ use candid::{Nat, Principal}; use clap::{ArgGroup, Args, Parser}; use ic_management_canister_types::CanisterSettings as MgmtCanisterSettings; use icp::canister::resolve_controllers; -use icp::identity::IdentitySelection; +use icp::host::EnvironmentSelection; use icp::parsers::{CyclesAmount, DurationAmount, MemoryAmount, parse_token_amount}; use icp::store_id::IdMapping; use icp::{Canister, host::CanisterSelection, prelude::*}; -use icp::{ - context::{Context, NetworkSelection}, - host::EnvironmentSelection, -}; +use icp_app::context::{Context, NetworkSelection}; +use icp_app::identity::IdentitySelection; use serde::Serialize; use tracing::{info, warn}; diff --git a/crates/icp-cli/src/commands/canister/delete.rs b/crates/icp-cli/src/commands/canister/delete.rs index 7643ce9ea..076e5f5b6 100644 --- a/crates/icp-cli/src/commands/canister/delete.rs +++ b/crates/icp-cli/src/commands/canister/delete.rs @@ -2,7 +2,8 @@ use anyhow::anyhow; use candid::Principal; use clap::Args; use ic_management_canister_types::CanisterIdRecord; -use icp::{context::Context, host::CanisterSelection}; +use icp::host::CanisterSelection; +use icp_app::context::Context; use icp::operations::{proxy_management, recover_cycles}; diff --git a/crates/icp-cli/src/commands/canister/install.rs b/crates/icp-cli/src/commands/canister/install.rs index a765f0399..d93a8ec32 100644 --- a/crates/icp-cli/src/commands/canister/install.rs +++ b/crates/icp-cli/src/commands/canister/install.rs @@ -6,8 +6,9 @@ use clap::{Args, ValueHint}; use dialoguer::Confirm; use ic_management_canister_types::CanisterInstallMode; use icp::fs; +use icp::host::CanisterSelection; use icp::prelude::*; -use icp::{context::Context, host::CanisterSelection}; +use icp_app::context::Context; use tracing::{info, warn}; use icp::operations::{ diff --git a/crates/icp-cli/src/commands/canister/link.rs b/crates/icp-cli/src/commands/canister/link.rs index 03f99d635..f83143d91 100644 --- a/crates/icp-cli/src/commands/canister/link.rs +++ b/crates/icp-cli/src/commands/canister/link.rs @@ -2,7 +2,8 @@ use anyhow::bail; use candid::Principal; use clap::Args; use clap_complete::ArgValueCandidates; -use icp::{context::Context, host::EnvironmentSelection}; +use icp::host::EnvironmentSelection; +use icp_app::context::Context; use tracing::info; use crate::options::EnvironmentOpt; diff --git a/crates/icp-cli/src/commands/canister/list.rs b/crates/icp-cli/src/commands/canister/list.rs index 456718640..0990aec2a 100644 --- a/crates/icp-cli/src/commands/canister/list.rs +++ b/crates/icp-cli/src/commands/canister/list.rs @@ -1,7 +1,7 @@ use std::io::stdout; use clap::Args; -use icp::context::Context; +use icp_app::context::Context; use serde::Serialize; use crate::options::EnvironmentOpt; diff --git a/crates/icp-cli/src/commands/canister/logs.rs b/crates/icp-cli/src/commands/canister/logs.rs index 49efb7279..ff8aaad31 100644 --- a/crates/icp-cli/src/commands/canister/logs.rs +++ b/crates/icp-cli/src/commands/canister/logs.rs @@ -5,8 +5,8 @@ use candid::Principal; use clap::Args; use ic_agent::Agent; use ic_management_canister_types::{CanisterLogFilter, CanisterLogRecord, FetchCanisterLogsArgs}; -use icp::context::Context; use icp::signal::stop_signal; +use icp_app::context::Context; use itertools::Itertools; use serde::Serialize; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; diff --git a/crates/icp-cli/src/commands/canister/metadata.rs b/crates/icp-cli/src/commands/canister/metadata.rs index c4ac92335..dcf2a2995 100644 --- a/crates/icp-cli/src/commands/canister/metadata.rs +++ b/crates/icp-cli/src/commands/canister/metadata.rs @@ -2,7 +2,7 @@ use std::io::stdout; use anyhow::bail; use clap::Args; -use icp::context::Context; +use icp_app::context::Context; use serde::Serialize; use crate::commands::args; diff --git a/crates/icp-cli/src/commands/canister/migrate_id.rs b/crates/icp-cli/src/commands/canister/migrate_id.rs index d0f109abd..1050bfb9e 100644 --- a/crates/icp-cli/src/commands/canister/migrate_id.rs +++ b/crates/icp-cli/src/commands/canister/migrate_id.rs @@ -8,7 +8,7 @@ use dialoguer::Confirm; use ic_management_canister_types::{ CanisterIdRecord, CanisterSettings, CanisterStatusType, UpdateSettingsArgs, }; -use icp::context::Context; +use icp_app::context::Context; use icp_canister_interfaces::nns_migration::{MigrationStatus, NNS_MIGRATION_PRINCIPAL}; use indicatif::{ProgressBar, ProgressStyle}; use num_traits::ToPrimitive; @@ -16,11 +16,11 @@ use tracing::{info, warn}; use crate::commands::args::{self, Canister}; use icp::host::CanisterSelection; -use icp::operations::canister_migration::{ - get_subnet_for_canister, migrate_canister, migration_status, -}; use icp::operations::misc::format_timestamp; use icp::operations::proxy_management; +use icp_app::operations::canister_migration::{ + get_subnet_for_canister, migrate_canister, migration_status, +}; /// Minimum cycles required for migration (10T). const MIN_CYCLES_FOR_MIGRATION: u128 = 10_000_000_000_000; diff --git a/crates/icp-cli/src/commands/canister/settings/show.rs b/crates/icp-cli/src/commands/canister/settings/show.rs index e405ed6a3..9e9a337cb 100644 --- a/crates/icp-cli/src/commands/canister/settings/show.rs +++ b/crates/icp-cli/src/commands/canister/settings/show.rs @@ -1,7 +1,7 @@ use clap::Args; use ic_agent::export::Principal; use ic_management_canister_types::{CanisterIdRecord, DefiniteCanisterSettings}; -use icp::context::Context; +use icp_app::context::Context; use std::fmt::Write; use icp::operations::proxy_management; diff --git a/crates/icp-cli/src/commands/canister/settings/sync.rs b/crates/icp-cli/src/commands/canister/settings/sync.rs index 1b42152e5..6a7e2ebca 100644 --- a/crates/icp-cli/src/commands/canister/settings/sync.rs +++ b/crates/icp-cli/src/commands/canister/settings/sync.rs @@ -1,7 +1,8 @@ use anyhow::bail; use candid::Principal; use clap::Args; -use icp::{context::Context, host::CanisterSelection}; +use icp::host::CanisterSelection; +use icp_app::context::Context; use tracing::warn; use crate::commands::args::CanisterCommandArgs; diff --git a/crates/icp-cli/src/commands/canister/settings/update.rs b/crates/icp-cli/src/commands/canister/settings/update.rs index f0cc4b3cb..fd54c57cd 100644 --- a/crates/icp-cli/src/commands/canister/settings/update.rs +++ b/crates/icp-cli/src/commands/canister/settings/update.rs @@ -10,8 +10,9 @@ use ic_management_canister_types::{ }; use icp::ProjectLoadError; use icp::canister::Visibility; +use icp::host::CanisterSelection; use icp::parsers::{CyclesAmount, DurationAmount, MemoryAmount}; -use icp::{context::Context, host::CanisterSelection}; +use icp_app::context::Context; use std::collections::{HashMap, HashSet}; use tracing::warn; diff --git a/crates/icp-cli/src/commands/canister/snapshot/create.rs b/crates/icp-cli/src/commands/canister/snapshot/create.rs index dbd561fb4..87d473cb7 100644 --- a/crates/icp-cli/src/commands/canister/snapshot/create.rs +++ b/crates/icp-cli/src/commands/canister/snapshot/create.rs @@ -7,7 +7,7 @@ use clap::Args; use ic_management_canister_types::{ CanisterIdRecord, CanisterStatusType, TakeCanisterSnapshotArgs, }; -use icp::context::Context; +use icp_app::context::Context; use serde::Serialize; use super::SnapshotId; diff --git a/crates/icp-cli/src/commands/canister/snapshot/delete.rs b/crates/icp-cli/src/commands/canister/snapshot/delete.rs index c72976a63..8e355222b 100644 --- a/crates/icp-cli/src/commands/canister/snapshot/delete.rs +++ b/crates/icp-cli/src/commands/canister/snapshot/delete.rs @@ -1,7 +1,7 @@ use candid::Principal; use clap::Args; use ic_management_canister_types::DeleteCanisterSnapshotArgs; -use icp::context::Context; +use icp_app::context::Context; use tracing::info; use super::SnapshotId; diff --git a/crates/icp-cli/src/commands/canister/snapshot/download.rs b/crates/icp-cli/src/commands/canister/snapshot/download.rs index 7c6b829df..8cc449e93 100644 --- a/crates/icp-cli/src/commands/canister/snapshot/download.rs +++ b/crates/icp-cli/src/commands/canister/snapshot/download.rs @@ -1,8 +1,8 @@ use byte_unit::{Byte, UnitType}; use candid::Principal; use clap::{Args, ValueHint}; -use icp::context::Context; use icp::prelude::*; +use icp_app::context::Context; use tracing::info; use icp::operations::task::{Task, TransferBlob, TransferDirection}; @@ -11,7 +11,7 @@ use super::SnapshotId; use crate::commands::args; use crate::render::rendered_task; use icp::operations::misc::format_timestamp; -use icp::operations::snapshot_transfer::{ +use icp_app::operations::snapshot_transfer::{ BlobType, SnapshotPaths, SnapshotTransferError, delete_download_progress, download_blob_to_file, download_wasm_chunk, load_download_progress, load_metadata, read_snapshot_metadata, save_metadata, diff --git a/crates/icp-cli/src/commands/canister/snapshot/list.rs b/crates/icp-cli/src/commands/canister/snapshot/list.rs index 67d7ae090..27c0350b5 100644 --- a/crates/icp-cli/src/commands/canister/snapshot/list.rs +++ b/crates/icp-cli/src/commands/canister/snapshot/list.rs @@ -4,7 +4,7 @@ use byte_unit::{Byte, UnitType}; use candid::Principal; use clap::Args; use ic_management_canister_types::CanisterIdRecord; -use icp::context::Context; +use icp_app::context::Context; use itertools::Itertools; use serde::Serialize; diff --git a/crates/icp-cli/src/commands/canister/snapshot/restore.rs b/crates/icp-cli/src/commands/canister/snapshot/restore.rs index 04713cae0..c0c523ff3 100644 --- a/crates/icp-cli/src/commands/canister/snapshot/restore.rs +++ b/crates/icp-cli/src/commands/canister/snapshot/restore.rs @@ -4,7 +4,7 @@ use clap::Args; use ic_management_canister_types::{ CanisterIdRecord, CanisterStatusType, LoadCanisterSnapshotArgs, }; -use icp::context::Context; +use icp_app::context::Context; use tracing::info; use super::SnapshotId; diff --git a/crates/icp-cli/src/commands/canister/snapshot/upload.rs b/crates/icp-cli/src/commands/canister/snapshot/upload.rs index 3cf6c654c..07943fa74 100644 --- a/crates/icp-cli/src/commands/canister/snapshot/upload.rs +++ b/crates/icp-cli/src/commands/canister/snapshot/upload.rs @@ -3,8 +3,8 @@ use std::io::stdout; use byte_unit::{Byte, UnitType}; use candid::Principal; use clap::{Args, ValueHint}; -use icp::context::Context; use icp::prelude::*; +use icp_app::context::Context; use serde::Serialize; use tracing::info; @@ -14,7 +14,7 @@ use super::SnapshotId; use crate::commands::args; use crate::render::rendered_task; use icp::operations::misc::format_timestamp; -use icp::operations::snapshot_transfer::{ +use icp_app::operations::snapshot_transfer::{ BlobType, SnapshotPaths, SnapshotTransferError, UploadProgress, delete_upload_progress, load_metadata, load_upload_progress, save_upload_progress, upload_blob_from_file, upload_snapshot_metadata, upload_wasm_chunk, diff --git a/crates/icp-cli/src/commands/canister/start.rs b/crates/icp-cli/src/commands/canister/start.rs index e9739d1f3..37dad36f7 100644 --- a/crates/icp-cli/src/commands/canister/start.rs +++ b/crates/icp-cli/src/commands/canister/start.rs @@ -1,7 +1,7 @@ use candid::Principal; use clap::Args; use ic_management_canister_types::CanisterIdRecord; -use icp::context::Context; +use icp_app::context::Context; use crate::commands::args; use icp::operations::proxy_management; diff --git a/crates/icp-cli/src/commands/canister/status.rs b/crates/icp-cli/src/commands/canister/status.rs index 6e9752237..01ecfa892 100644 --- a/crates/icp-cli/src/commands/canister/status.rs +++ b/crates/icp-cli/src/commands/canister/status.rs @@ -5,8 +5,10 @@ use ic_agent::{Agent, AgentError, agent::RejectResponse, export::Principal}; use ic_management_canister_types::{CanisterIdRecord, CanisterStatusResult, EnvironmentVariable}; use icp::{ canister::Visibility, - context::{Context, NetworkSelection}, host::{CanisterSelection, EnvironmentSelection}, +}; +use icp_app::{ + context::{Context, NetworkSelection}, identity::IdentitySelection, }; use serde::Serialize; diff --git a/crates/icp-cli/src/commands/canister/stop.rs b/crates/icp-cli/src/commands/canister/stop.rs index b19489310..39388b183 100644 --- a/crates/icp-cli/src/commands/canister/stop.rs +++ b/crates/icp-cli/src/commands/canister/stop.rs @@ -1,7 +1,7 @@ use candid::Principal; use clap::Args; use ic_management_canister_types::CanisterIdRecord; -use icp::context::Context; +use icp_app::context::Context; use crate::commands::args; use icp::operations::proxy_management; diff --git a/crates/icp-cli/src/commands/canister/top_up.rs b/crates/icp-cli/src/commands/canister/top_up.rs index bfab59bb8..537e6cdf5 100644 --- a/crates/icp-cli/src/commands/canister/top_up.rs +++ b/crates/icp-cli/src/commands/canister/top_up.rs @@ -2,15 +2,15 @@ use anyhow::{Context as _, bail}; use bigdecimal::BigDecimal; use candid::{Decode, Encode, Nat}; use clap::Args; -use icp::context::Context; use icp::parsers::CyclesAmount; +use icp_app::context::Context; use icp_canister_interfaces::cycles_ledger::{ CYCLES_LEDGER_PRINCIPAL, WithdrawArgs, WithdrawResponse, }; use tracing::info; use crate::commands::args; -use icp::operations::token::TokenAmount; +use icp_app::operations::token::TokenAmount; /// Top up a canister with cycles #[derive(Debug, Args)] diff --git a/crates/icp-cli/src/commands/cycles/balance.rs b/crates/icp-cli/src/commands/cycles/balance.rs index 9c088a73f..834de274d 100644 --- a/crates/icp-cli/src/commands/cycles/balance.rs +++ b/crates/icp-cli/src/commands/cycles/balance.rs @@ -3,14 +3,14 @@ use std::io::stdout; use bigdecimal::BigDecimal; use candid::Principal; use clap::Args; -use icp::context::Context; +use icp_app::context::Context; use icp_canister_interfaces::cycles_ledger::CYCLES_LEDGER_PRINCIPAL; use serde::Serialize; use crate::commands::args::TokenCommandArgs; use crate::commands::parsers::parse_subaccount; -use icp::operations::token::TokenAmount; -use icp::operations::token::balance::get_raw_balance; +use icp_app::operations::token::TokenAmount; +use icp_app::operations::token::balance::get_raw_balance; /// Display the cycles balance #[derive(Args, Clone, Debug)] diff --git a/crates/icp-cli/src/commands/cycles/mint.rs b/crates/icp-cli/src/commands/cycles/mint.rs index 69e009f0a..2b0993863 100644 --- a/crates/icp-cli/src/commands/cycles/mint.rs +++ b/crates/icp-cli/src/commands/cycles/mint.rs @@ -3,13 +3,13 @@ use std::io::stdout; use anyhow::bail; use bigdecimal::BigDecimal; use clap::Args; -use icp::context::Context; use icp::parsers::{CyclesAmount, parse_token_amount}; +use icp_app::context::Context; use serde::Serialize; use crate::commands::args::TokenCommandArgs; use crate::commands::parsers::parse_subaccount; -use icp::operations::token::mint::mint_cycles; +use icp_app::operations::token::mint::mint_cycles; /// Convert ICP to cycles. /// diff --git a/crates/icp-cli/src/commands/cycles/transfer.rs b/crates/icp-cli/src/commands/cycles/transfer.rs index 17f70b3dd..e8091ebba 100644 --- a/crates/icp-cli/src/commands/cycles/transfer.rs +++ b/crates/icp-cli/src/commands/cycles/transfer.rs @@ -2,15 +2,15 @@ use std::io::stdout; use anyhow::ensure; use clap::Args; -use icp::context::Context; use icp::parsers::CyclesAmount; +use icp_app::context::Context; use icp_canister_interfaces::cycles_ledger::{CYCLES_LEDGER_BLOCK_FEE, CYCLES_LEDGER_PRINCIPAL}; use icrc_ledger_types::icrc1::account::Account; use serde::Serialize; use crate::commands::args::TokenCommandArgs; use crate::commands::parsers::parse_subaccount; -use icp::operations::token::transfer::icrc1_transfer; +use icp_app::operations::token::transfer::icrc1_transfer; /// Transfer cycles to another principal #[derive(Debug, Args)] diff --git a/crates/icp-cli/src/commands/deploy.rs b/crates/icp-cli/src/commands/deploy.rs index 2400c4fca..16c027c63 100644 --- a/crates/icp-cli/src/commands/deploy.rs +++ b/crates/icp-cli/src/commands/deploy.rs @@ -7,11 +7,10 @@ use icp::operations::deploy::{DeployParams, DeployReport, deploy, resolve_target use icp::parsers::CyclesAmount; use icp::{ agent::LazyAgent, - context::Context, host::{CanisterSelection, EnvironmentSelection}, - identity::IdentitySelection, network::Configuration as NetworkConfiguration, }; +use icp_app::{context::Context, identity::IdentitySelection}; use icp_canister_interfaces::candid_ui::MAINNET_CANDID_UI_CID; use serde::Serialize; use tracing::info; @@ -124,7 +123,6 @@ pub(crate) async fn exec(ctx: &Context, args: &DeployArgs) -> Result<(), anyhow: // creates it. let (identity, environment) = (&identity_selection, &environment_selection); let agent = LazyAgent::new(move || ctx.get_agent_for_env(identity, environment)); - let pkg_cache = ctx.dirs.package_cache()?; let params = DeployParams { environment: environment_selection.clone(), @@ -143,15 +141,7 @@ pub(crate) async fn exec(ctx: &Context, args: &DeployArgs) -> Result<(), anyhow: // command having to await it phase by phase. let mut report = DeployReport::default(); let result = rendered(ctx.debug, async |reporter| { - deploy( - &ctx.host, - &agent, - &pkg_cache, - ¶ms, - reporter, - &mut report, - ) - .await + deploy(&ctx.host, &agent, ¶ms, reporter, &mut report).await }) .await; @@ -249,7 +239,7 @@ async fn print_canister_urls( canister_names: &[String], json: bool, ) -> Result<(), anyhow::Error> { - use icp::network::custom_domains::{canister_gateway_url, gateway_domain}; + use icp_app::network::custom_domains::{canister_gateway_url, gateway_domain}; let env = ctx.host.get_environment(environment_selection).await?; @@ -415,7 +405,7 @@ async fn get_candid_ui_id( match &env.network.configuration { NetworkConfiguration::Managed { managed: _ } => { // Try to get the candid UI ID from the network descriptor - let nd = ctx.host.network.get_network_directory(&env.network).ok()?; + let nd = ctx.network_dirs.get_network_directory(&env.network).ok()?; if let Ok(Some(desc)) = nd.load_network_descriptor().await && let Some(candid_ui) = desc.candid_ui_canister_id { diff --git a/crates/icp-cli/src/commands/environment/list.rs b/crates/icp-cli/src/commands/environment/list.rs index 11d16ad87..c2f62f23b 100644 --- a/crates/icp-cli/src/commands/environment/list.rs +++ b/crates/icp-cli/src/commands/environment/list.rs @@ -1,5 +1,5 @@ use clap::Args; -use icp::context::Context; +use icp_app::context::Context; /// List the environments defined in this project, one per line. /// diff --git a/crates/icp-cli/src/commands/identity/account_id.rs b/crates/icp-cli/src/commands/identity/account_id.rs index e1e0be17d..a1a9c412d 100644 --- a/crates/icp-cli/src/commands/identity/account_id.rs +++ b/crates/icp-cli/src/commands/identity/account_id.rs @@ -1,7 +1,7 @@ use candid::Principal; use clap::{Args, ValueEnum}; use ic_ledger_types::{AccountIdentifier, Subaccount}; -use icp::context::Context; +use icp_app::context::Context; use icrc_ledger_types::icrc1::account::Account; use crate::commands::parsers::parse_subaccount; diff --git a/crates/icp-cli/src/commands/identity/default.rs b/crates/icp-cli/src/commands/identity/default.rs index d7235c348..4e9bfab18 100644 --- a/crates/icp-cli/src/commands/identity/default.rs +++ b/crates/icp-cli/src/commands/identity/default.rs @@ -1,7 +1,7 @@ use clap::Args; use clap_complete::ArgValueCandidates; -use icp::context::Context; -use icp::identity::manifest::{IdentityDefaults, IdentityList, change_default_identity}; +use icp_app::context::Context; +use icp_app::identity::manifest::{IdentityDefaults, IdentityList, change_default_identity}; use tracing::info; /// Display or set the currently selected identity diff --git a/crates/icp-cli/src/commands/identity/delegation/request.rs b/crates/icp-cli/src/commands/identity/delegation/request.rs index f095a7e0c..97743feaf 100644 --- a/crates/icp-cli/src/commands/identity/delegation/request.rs +++ b/crates/icp-cli/src/commands/identity/delegation/request.rs @@ -1,7 +1,8 @@ use clap::{Args, ValueHint}; use dialoguer::Password; use elliptic_curve::zeroize::Zeroizing; -use icp::{context::Context, fs::read_to_string, identity::key, prelude::*}; +use icp::{fs::read_to_string, prelude::*}; +use icp_app::{context::Context, identity::key}; use pem::Pem; use snafu::{ResultExt, Snafu}; use tracing::warn; diff --git a/crates/icp-cli/src/commands/identity/delegation/sign.rs b/crates/icp-cli/src/commands/identity/delegation/sign.rs index 6c2c23180..5833e1e7e 100644 --- a/crates/icp-cli/src/commands/identity/delegation/sign.rs +++ b/crates/icp-cli/src/commands/identity/delegation/sign.rs @@ -5,13 +5,12 @@ use std::{ use clap::{Args, ValueHint}; use ic_agent::{Identity as _, export::Principal, identity::Delegation as AgentDelegation}; -use icp::{ +use icp::{fs::read_to_string, prelude::*}; +use icp_app::{ context::{Context, GetIdentityError}, - fs::read_to_string, identity::delegation::{ Delegation as WireDelegation, DelegationChain, SignedDelegation as WireSignedDelegation, }, - prelude::*, }; use pem::Pem; use snafu::{OptionExt, ResultExt, Snafu}; diff --git a/crates/icp-cli/src/commands/identity/delegation/use.rs b/crates/icp-cli/src/commands/identity/delegation/use.rs index c4a193dfb..a8feaa633 100644 --- a/crates/icp-cli/src/commands/identity/delegation/use.rs +++ b/crates/icp-cli/src/commands/identity/delegation/use.rs @@ -1,14 +1,13 @@ use clap::{Args, ValueHint}; use clap_complete::ArgValueCandidates; -use icp::{ +use icp::{fs::json, prelude::*}; +use icp_app::{ context::Context, - fs::json, identity::{ delegation::DelegationChain, key, manifest::{DelegationKeyStorage, PemFormat}, }, - prelude::*, }; use snafu::{ResultExt, Snafu}; use tracing::{info, warn}; diff --git a/crates/icp-cli/src/commands/identity/delete.rs b/crates/icp-cli/src/commands/identity/delete.rs index 967d0bab0..6b824b21c 100644 --- a/crates/icp-cli/src/commands/identity/delete.rs +++ b/crates/icp-cli/src/commands/identity/delete.rs @@ -1,7 +1,7 @@ use clap::Args; use clap_complete::ArgValueCandidates; -use icp::context::Context; -use icp::identity::key::delete_identity; +use icp_app::context::Context; +use icp_app::identity::key::delete_identity; use tracing::info; /// Delete an identity diff --git a/crates/icp-cli/src/commands/identity/export.rs b/crates/icp-cli/src/commands/identity/export.rs index cfb56338b..073d20469 100644 --- a/crates/icp-cli/src/commands/identity/export.rs +++ b/crates/icp-cli/src/commands/identity/export.rs @@ -3,10 +3,10 @@ use clap::{Args, ValueHint}; use clap_complete::ArgValueCandidates; use dialoguer::Password; use elliptic_curve::zeroize::Zeroizing; -use icp::context::Context; use icp::fs::read_to_string; -use icp::identity::key::{ExportFormat, export_identity}; use icp::prelude::*; +use icp_app::context::Context; +use icp_app::identity::key::{ExportFormat, export_identity}; /// Print the PEM file for the identity #[derive(Debug, Args)] diff --git a/crates/icp-cli/src/commands/identity/import.rs b/crates/icp-cli/src/commands/identity/import.rs index 28dc343ae..14cbda4af 100644 --- a/crates/icp-cli/src/commands/identity/import.rs +++ b/crates/icp-cli/src/commands/identity/import.rs @@ -2,16 +2,16 @@ use bip39::{Language, Mnemonic}; use clap::{ArgGroup, Args, ValueHint}; use dialoguer::Password; use elliptic_curve::zeroize::Zeroizing; -use icp::identity::{ +use icp::{ + fs::{json, read_to_string}, + prelude::*, +}; +use icp_app::identity::{ delegation::DelegationChain, key::{CreateFormat, CreateIdentityError, IdentityKey, create_identity}, manifest::IdentityKeyAlgorithm, seed::derive_key_from_seed_slip10, }; -use icp::{ - fs::{json, read_to_string}, - prelude::*, -}; use itertools::Itertools; use k256::Secp256k1; use p256::NistP256; @@ -24,7 +24,7 @@ use sec1::{EcParameters, EcPrivateKey}; use snafu::{OptionExt, ResultExt, Snafu, ensure}; use tracing::{info, warn}; -use icp::context::Context; +use icp_app::context::Context; use crate::commands::identity::StorageMode; diff --git a/crates/icp-cli/src/commands/identity/link/hsm.rs b/crates/icp-cli/src/commands/identity/link/hsm.rs index 5b491068e..5ec41b02d 100644 --- a/crates/icp-cli/src/commands/identity/link/hsm.rs +++ b/crates/icp-cli/src/commands/identity/link/hsm.rs @@ -1,9 +1,9 @@ use clap::{Args, ValueHint}; use dialoguer::Password; -use icp::{ +use icp::prelude::*; +use icp_app::{ context::Context, identity::{key::link_hsm_identity, manifest::IdentityList}, - prelude::*, }; use snafu::{ResultExt, Snafu, ensure}; use tracing::info; @@ -88,7 +88,7 @@ pub(crate) enum HsmError { #[snafu(display("failed to load identity list"))] LoadIdentityList { - source: icp::identity::manifest::LoadIdentityManifestError, + source: icp_app::identity::manifest::LoadIdentityManifestError, }, #[snafu(transparent)] @@ -96,6 +96,6 @@ pub(crate) enum HsmError { #[snafu(display("failed to link HSM identity"))] LinkHsm { - source: icp::identity::key::LinkHsmIdentityError, + source: icp_app::identity::key::LinkHsmIdentityError, }, } diff --git a/crates/icp-cli/src/commands/identity/link/web.rs b/crates/icp-cli/src/commands/identity/link/web.rs index 330fe2fcc..9fd8b14da 100644 --- a/crates/icp-cli/src/commands/identity/link/web.rs +++ b/crates/icp-cli/src/commands/identity/link/web.rs @@ -13,15 +13,14 @@ use clap::{Args, ValueHint}; use dialoguer::Password; use elliptic_curve::zeroize::Zeroizing; use ic_agent::{Identity as _, export::Principal, identity::BasicIdentity}; -use icp::{ +use icp::{fs::read_to_string, prelude::*}; +use icp_app::{ context::Context, - fs::read_to_string, identity::{ delegation::DelegationChain, key::{self, validate_password}, manifest::IdentityList, }, - prelude::*, }; use indicatif::{ProgressBar, ProgressStyle}; use rand::RngExt as _; @@ -164,7 +163,7 @@ pub(crate) enum WebAuthError { #[snafu(display("failed to load identity list"))] LoadIdentityList { - source: icp::identity::manifest::LoadIdentityManifestError, + source: icp_app::identity::manifest::LoadIdentityManifestError, }, #[snafu(display("failed to read storage password file"))] diff --git a/crates/icp-cli/src/commands/identity/list.rs b/crates/icp-cli/src/commands/identity/list.rs index a7ecaa116..746ee2de3 100644 --- a/crates/icp-cli/src/commands/identity/list.rs +++ b/crates/icp-cli/src/commands/identity/list.rs @@ -2,11 +2,11 @@ use std::io::stdout; use candid::Principal; use clap::Args; -use icp::identity::manifest::{IdentityDefaults, IdentityList}; +use icp_app::identity::manifest::{IdentityDefaults, IdentityList}; use itertools::Itertools; use serde::Serialize; -use icp::context::Context; +use icp_app::context::Context; /// List the identities #[derive(Debug, Args)] diff --git a/crates/icp-cli/src/commands/identity/new.rs b/crates/icp-cli/src/commands/identity/new.rs index 524e6ce19..9c61e19ff 100644 --- a/crates/icp-cli/src/commands/identity/new.rs +++ b/crates/icp-cli/src/commands/identity/new.rs @@ -5,17 +5,14 @@ use bip39::{Language, Mnemonic, MnemonicType}; use clap::{Args, ValueHint}; use dialoguer::Password; use elliptic_curve::zeroize::Zeroizing; -use icp::{ - fs::write_string, - identity::{ - key::{CreateFormat, create_identity, validate_password}, - manifest::{IdentityKeyAlgorithm, IdentityList}, - seed::derive_key_from_seed_slip10, - }, - prelude::*, +use icp::{fs::write_string, prelude::*}; +use icp_app::identity::{ + key::{CreateFormat, create_identity, validate_password}, + manifest::{IdentityKeyAlgorithm, IdentityList}, + seed::derive_key_from_seed_slip10, }; -use icp::context::Context; +use icp_app::context::Context; use serde::Serialize; use tracing::{info, warn}; diff --git a/crates/icp-cli/src/commands/identity/principal.rs b/crates/icp-cli/src/commands/identity/principal.rs index b276912dc..79c93f748 100644 --- a/crates/icp-cli/src/commands/identity/principal.rs +++ b/crates/icp-cli/src/commands/identity/principal.rs @@ -1,5 +1,5 @@ use clap::Args; -use icp::context::Context; +use icp_app::context::Context; use crate::options::IdentityOpt; diff --git a/crates/icp-cli/src/commands/identity/reauth.rs b/crates/icp-cli/src/commands/identity/reauth.rs index 937cd1f2c..da1754187 100644 --- a/crates/icp-cli/src/commands/identity/reauth.rs +++ b/crates/icp-cli/src/commands/identity/reauth.rs @@ -2,7 +2,7 @@ use std::time::Duration; use clap::Args; use clap_complete::ArgValueCandidates; -use icp::{ +use icp_app::{ context::Context, identity::{ key, @@ -140,12 +140,12 @@ pub(crate) enum LoginError { #[snafu(transparent)] LoadManifest { - source: icp::identity::manifest::LoadIdentityManifestError, + source: icp_app::identity::manifest::LoadIdentityManifestError, }, #[snafu(transparent)] LoadSettings { - source: icp::settings::LoadSettingsError, + source: icp_app::settings::LoadSettingsError, }, #[snafu(display("no identity found with name `{name}`"))] diff --git a/crates/icp-cli/src/commands/identity/rename.rs b/crates/icp-cli/src/commands/identity/rename.rs index a3b112361..5b9f02572 100644 --- a/crates/icp-cli/src/commands/identity/rename.rs +++ b/crates/icp-cli/src/commands/identity/rename.rs @@ -1,7 +1,7 @@ use clap::Args; use clap_complete::ArgValueCandidates; -use icp::context::Context; -use icp::identity::key::rename_identity; +use icp_app::context::Context; +use icp_app::identity::key::rename_identity; use tracing::info; /// Rename an identity diff --git a/crates/icp-cli/src/commands/message/send.rs b/crates/icp-cli/src/commands/message/send.rs index 1d0ebecb6..0e48aa191 100644 --- a/crates/icp-cli/src/commands/message/send.rs +++ b/crates/icp-cli/src/commands/message/send.rs @@ -2,11 +2,12 @@ use anyhow::{Context as _, bail}; use candid::{IDLArgs, TypeEnv, types::Function}; use clap::{Args, ValueHint}; use ic_agent::agent::CallResponse; -use icp::context::{Context, IC_ROOT_KEY}; -use icp::identity::IdentitySelection; use icp::network::RootKeySpec; +use icp::prelude::IC_ROOT_KEY; use icp::prelude::*; -use icp::signed_message::{ +use icp_app::context::Context; +use icp_app::identity::IdentitySelection; +use icp_app::signed_message::{ CallType, Destination, SUBMISSION_WINDOW, SignedMessage, Validated, WindowState, format_timestamp, }; diff --git a/crates/icp-cli/src/commands/network/args.rs b/crates/icp-cli/src/commands/network/args.rs index c697b2c99..44a8a7d10 100644 --- a/crates/icp-cli/src/commands/network/args.rs +++ b/crates/icp-cli/src/commands/network/args.rs @@ -1,6 +1,7 @@ use clap::Args; use clap_complete::ArgValueCandidates; -use icp::{context::NetworkOrEnvironmentSelection, prelude::LOCAL}; +use icp::prelude::LOCAL; +use icp_app::context::NetworkOrEnvironmentSelection; #[derive(Args, Clone, Debug)] pub(crate) struct NetworkOrEnvironmentArgs { diff --git a/crates/icp-cli/src/commands/network/list.rs b/crates/icp-cli/src/commands/network/list.rs index 433511795..a66a7ea7d 100644 --- a/crates/icp-cli/src/commands/network/list.rs +++ b/crates/icp-cli/src/commands/network/list.rs @@ -1,5 +1,5 @@ use clap::Args; -use icp::context::Context; +use icp_app::context::Context; /// List all networks configured in the project #[derive(Args, Debug)] diff --git a/crates/icp-cli/src/commands/network/ping.rs b/crates/icp-cli/src/commands/network/ping.rs index d26086c7b..37b5b5be6 100644 --- a/crates/icp-cli/src/commands/network/ping.rs +++ b/crates/icp-cli/src/commands/network/ping.rs @@ -1,7 +1,7 @@ use anyhow::bail; use clap::Args; use ic_agent::{Agent, agent::status::Status}; -use icp::{context::Context, identity::IdentitySelection}; +use icp_app::{context::Context, identity::IdentitySelection}; use std::time::Duration; use tokio::time::sleep; use tracing::info; diff --git a/crates/icp-cli/src/commands/network/start.rs b/crates/icp-cli/src/commands/network/start.rs index 9d247bf56..dbe880a23 100644 --- a/crates/icp-cli/src/commands/network/start.rs +++ b/crates/icp-cli/src/commands/network/start.rs @@ -3,12 +3,12 @@ use std::sync::{Arc, OnceLock}; use anyhow::{Context as _, bail}; use candid::Principal; use clap::Args; +use icp::network::Configuration; use icp::network::ManagedMode; use icp::prelude::*; -use icp::{ +use icp_app::{ identity::manifest::IdentityList, network::{ - Configuration, managed::{ cache::{ check_launcher_update_available, download_launcher_version, @@ -25,7 +25,7 @@ use tracing::{debug, info, warn}; use crate::render::{ProgressManager, ProgressManagerSettings}; use super::args::NetworkOrEnvironmentArgs; -use icp::context::Context; +use icp_app::context::Context; /// Run a given network. /// @@ -87,7 +87,7 @@ pub(crate) async fn exec(ctx: &Context, args: &StartArgs) -> Result<(), anyhow:: let pdir = &p.dir; // Network directory - let nd = ctx.host.network.get_network_directory(&network)?; + let nd = ctx.network_dirs.get_network_directory(&network)?; nd.ensure_exists() .context("failed to create network directory")?; @@ -135,7 +135,7 @@ pub(crate) async fn exec(ctx: &Context, args: &StartArgs) -> Result<(), anyhow:: .identity()? .with_read(async |dirs| { let ids = IdentityList::load_from(dirs)?; - let defaults = icp::identity::manifest::IdentityDefaults::load_from(dirs)?; + let defaults = icp_app::identity::manifest::IdentityDefaults::load_from(dirs)?; Ok::<_, anyhow::Error>((ids, defaults)) }) .await??; diff --git a/crates/icp-cli/src/commands/network/status.rs b/crates/icp-cli/src/commands/network/status.rs index 9f010c370..8944ed32a 100644 --- a/crates/icp-cli/src/commands/network/status.rs +++ b/crates/icp-cli/src/commands/network/status.rs @@ -1,9 +1,7 @@ use anyhow::Context as _; use clap::Args; -use icp::{ - context::Context, - network::{Configuration, RootKeySource}, -}; +use icp::network::{Configuration, RootKeySource}; +use icp_app::context::Context; use serde::Serialize; use super::args::NetworkOrEnvironmentArgs; @@ -69,7 +67,7 @@ pub(crate) async fn exec(ctx: &Context, args: &StatusArgs) -> Result<(), anyhow: let status = match &network.configuration { Configuration::Managed { managed: _ } => { // Network directory - let nd = ctx.host.network.get_network_directory(&network)?; + let nd = ctx.network_dirs.get_network_directory(&network)?; // Load network descriptor let descriptor = nd diff --git a/crates/icp-cli/src/commands/network/stop.rs b/crates/icp-cli/src/commands/network/stop.rs index ec1f7911c..4613dd199 100644 --- a/crates/icp-cli/src/commands/network/stop.rs +++ b/crates/icp-cli/src/commands/network/stop.rs @@ -1,13 +1,11 @@ use anyhow::bail; use clap::Args; -use icp::{ - fs::remove_file, - network::{Configuration, config::ChildLocator, managed::run::stop_network}, -}; +use icp::{fs::remove_file, network::Configuration}; +use icp_app::network::{config::ChildLocator, managed::run::stop_network}; use tracing::info; use super::args::NetworkOrEnvironmentArgs; -use icp::context::Context; +use icp_app::context::Context; /// Stop a background network #[derive(Args, Debug)] @@ -47,7 +45,7 @@ pub async fn exec(ctx: &Context, cmd: &Cmd) -> Result<(), anyhow::Error> { }; // Network directory - let nd = ctx.host.network.get_network_directory(&network)?; + let nd = ctx.network_dirs.get_network_directory(&network)?; let descriptor = nd .load_network_descriptor() diff --git a/crates/icp-cli/src/commands/network/update.rs b/crates/icp-cli/src/commands/network/update.rs index f5e960843..40792251b 100644 --- a/crates/icp-cli/src/commands/network/update.rs +++ b/crates/icp-cli/src/commands/network/update.rs @@ -1,7 +1,7 @@ use std::sync::{Arc, OnceLock}; use clap::Parser; -use icp::{context::Context, network::managed::cache::download_launcher_version}; +use icp_app::{context::Context, network::managed::cache::download_launcher_version}; use crate::render::{ProgressManager, ProgressManagerSettings}; diff --git a/crates/icp-cli/src/commands/new.rs b/crates/icp-cli/src/commands/new.rs index 1c27a82bc..550d652da 100644 --- a/crates/icp-cli/src/commands/new.rs +++ b/crates/icp-cli/src/commands/new.rs @@ -197,7 +197,7 @@ fn resolve_name(args: &IcpGenerateArgs) -> Result, anyhow::Error> } pub(crate) async fn exec( - ctx: &icp::context::Context, + ctx: &icp_app::context::Context, args: &IcpGenerateArgs, ) -> Result<(), anyhow::Error> { // Check for conflicting flags: --quiet and --debug cannot be used together diff --git a/crates/icp-cli/src/commands/project/bundle.rs b/crates/icp-cli/src/commands/project/bundle.rs index 4d2bddd4d..774e11772 100644 --- a/crates/icp-cli/src/commands/project/bundle.rs +++ b/crates/icp-cli/src/commands/project/bundle.rs @@ -2,9 +2,9 @@ use std::collections::HashSet; use anyhow::Context as _; use clap::{Args, ValueHint}; -use icp::context::Context; use icp::host::EnvironmentSelection; use icp::prelude::*; +use icp_app::context::Context; use tracing::warn; use icp::operations::bundle::create_bundle; @@ -51,7 +51,6 @@ pub(crate) async fn exec(ctx: &Context, args: &BundleArgs) -> Result<(), anyhow: ); } - let pkg_cache = ctx.dirs.package_cache()?; rendered(ctx.debug, async |reporter| { create_bundle( &project.dir, @@ -60,7 +59,7 @@ pub(crate) async fn exec(ctx: &Context, args: &BundleArgs) -> Result<(), anyhow: &args.environment, ctx.host.builder.clone(), ctx.host.artifacts.clone(), - &pkg_cache, + ctx.host.wasm.as_ref(), reporter, &args.output, ) diff --git a/crates/icp-cli/src/commands/project/show.rs b/crates/icp-cli/src/commands/project/show.rs index 757b05764..9bb969b3b 100644 --- a/crates/icp-cli/src/commands/project/show.rs +++ b/crates/icp-cli/src/commands/project/show.rs @@ -1,7 +1,7 @@ use anyhow::Context as _; use clap::Args; -use icp::context::Context; +use icp_app::context::Context; /// Outputs the project's effective yaml configuration. /// diff --git a/crates/icp-cli/src/commands/settings.rs b/crates/icp-cli/src/commands/settings.rs index 147544b70..89135222e 100644 --- a/crates/icp-cli/src/commands/settings.rs +++ b/crates/icp-cli/src/commands/settings.rs @@ -1,7 +1,7 @@ use std::{fmt, str::FromStr}; use clap::{Args, Subcommand}; -use icp::{ +use icp_app::{ context::Context, settings::{Settings, UpdateCheck}, }; diff --git a/crates/icp-cli/src/commands/sync.rs b/crates/icp-cli/src/commands/sync.rs index b246f736a..813ced9bb 100644 --- a/crates/icp-cli/src/commands/sync.rs +++ b/crates/icp-cli/src/commands/sync.rs @@ -4,11 +4,9 @@ use clap::Args; use clap_complete::ArgValueCandidates; use futures::future::try_join_all; use ic_management_canister_types::{CanisterId, CanisterIdRecord, CanisterStatusType}; -use icp::identity::IdentitySelection; -use icp::{ - context::Context, - host::{CanisterSelection, EnvironmentSelection}, -}; +use icp::host::{CanisterSelection, EnvironmentSelection}; +use icp_app::context::Context; +use icp_app::identity::IdentitySelection; use std::collections::BTreeMap; use tracing::info; @@ -132,7 +130,6 @@ pub(crate) async fn exec(ctx: &Context, args: &SyncArgs) -> Result<(), anyhow::E .into_iter() .collect(); - let pkg_cache = ctx.dirs.package_cache()?; let project_dir = ctx.host.project.load().await?.dir; let urls = ctx.host.network.urls(&env.network).await?; @@ -147,7 +144,6 @@ pub(crate) async fn exec(ctx: &Context, args: &SyncArgs) -> Result<(), anyhow::E urls, canister_ids, args.proxy, - &pkg_cache, reporter, ) .await diff --git a/crates/icp-cli/src/commands/token/allowance.rs b/crates/icp-cli/src/commands/token/allowance.rs index 007679b90..c58f3e25d 100644 --- a/crates/icp-cli/src/commands/token/allowance.rs +++ b/crates/icp-cli/src/commands/token/allowance.rs @@ -2,14 +2,14 @@ use std::io::stdout; use candid::Principal; use clap::Args; -use icp::context::Context; +use icp_app::context::Context; use icrc_ledger_types::icrc1::account::Account; use serde::Serialize; use crate::commands::args::TokenCommandArgs; use crate::commands::parsers::parse_subaccount; use crate::commands::token::format_expiry; -use icp::operations::token::allowance::get_allowance; +use icp_app::operations::token::allowance::get_allowance; /// Display the allowance granted to a spender (ICRC-2) (default token: icp) /// diff --git a/crates/icp-cli/src/commands/token/approve.rs b/crates/icp-cli/src/commands/token/approve.rs index e4bdc9428..6fa50a545 100644 --- a/crates/icp-cli/src/commands/token/approve.rs +++ b/crates/icp-cli/src/commands/token/approve.rs @@ -4,8 +4,8 @@ use anyhow::Context as _; use bigdecimal::BigDecimal; use candid::Principal; use clap::Args; -use icp::context::Context; use icp::parsers::{DurationAmount, parse_token_amount}; +use icp_app::context::Context; use icrc_ledger_types::icrc1::account::Account; use serde::Serialize; use time::OffsetDateTime; @@ -13,7 +13,7 @@ use time::OffsetDateTime; use crate::commands::args::TokenCommandArgs; use crate::commands::parsers::parse_subaccount; use crate::commands::token::format_expiry; -use icp::operations::token::approve::approve; +use icp_app::operations::token::approve::approve; /// Approve a spender to transfer tokens on your behalf (ICRC-2) (default token: icp) /// diff --git a/crates/icp-cli/src/commands/token/balance.rs b/crates/icp-cli/src/commands/token/balance.rs index 819d1a875..b0f4b3174 100644 --- a/crates/icp-cli/src/commands/token/balance.rs +++ b/crates/icp-cli/src/commands/token/balance.rs @@ -2,12 +2,12 @@ use std::io::stdout; use candid::Principal; use clap::Args; -use icp::context::Context; +use icp_app::context::Context; use serde::Serialize; use crate::commands::args::TokenCommandArgs; use crate::commands::parsers::parse_subaccount; -use icp::operations::token::balance::get_balance; +use icp_app::operations::token::balance::get_balance; /// Display the token balance on the ledger (default token: icp) #[derive(Args, Clone, Debug)] diff --git a/crates/icp-cli/src/commands/token/transfer.rs b/crates/icp-cli/src/commands/token/transfer.rs index ca21f989c..8bfbf34fb 100644 --- a/crates/icp-cli/src/commands/token/transfer.rs +++ b/crates/icp-cli/src/commands/token/transfer.rs @@ -2,13 +2,13 @@ use std::io::stdout; use bigdecimal::BigDecimal; use clap::Args; -use icp::context::Context; use icp::parsers::parse_token_amount; +use icp_app::context::Context; use serde::Serialize; use crate::commands::args::{FlexibleAccountId, TokenCommandArgs}; use crate::commands::parsers::parse_subaccount; -use icp::operations::token::transfer::transfer; +use icp_app::operations::token::transfer::transfer; /// Transfer ICP or ICRC1 tokens through their ledger (default token: icp) #[derive(Debug, Args)] diff --git a/crates/icp-cli/src/complete.rs b/crates/icp-cli/src/complete.rs index 36b9ef6f5..0c4184d67 100644 --- a/crates/icp-cli/src/complete.rs +++ b/crates/icp-cli/src/complete.rs @@ -14,11 +14,11 @@ use std::time::Duration; use clap::CommandFactory as _; use clap_complete::CompleteEnv; use clap_complete::engine::CompletionCandidate; -use icp::context::Context; -use icp::identity::manifest::IdentityList; use icp::network::Configuration; use icp::prelude::*; use icp::{Environment, Network, Project}; +use icp_app::context::Context; +use icp_app::identity::manifest::IdentityList; /// Answer a completion request and exit, if this invocation is one. /// @@ -61,7 +61,7 @@ fn context() -> Option<&'static Context> { CONTEXT .get_or_init(|| { - icp::context::initialize( + icp_app::context::initialize( std::env::var("ICP_PROJECT_ROOT").ok().map(PathBuf::from), false, Arc::new(|| Err("cannot prompt while completing".to_string())), diff --git a/crates/icp-cli/src/dist.rs b/crates/icp-cli/src/dist.rs index 41fa58075..f7911d75b 100644 --- a/crates/icp-cli/src/dist.rs +++ b/crates/icp-cli/src/dist.rs @@ -2,7 +2,7 @@ use std::sync::LazyLock; use std::time::{Duration, SystemTime}; use axoupdater::AxoUpdater; -use icp::settings::UpdateCheck; +use icp_app::settings::UpdateCheck; use reqwest::Client; use tracing::debug; @@ -181,10 +181,10 @@ fn newer_than_current(version_str: &str) -> bool { const ONE_DAY: Duration = Duration::from_secs(24 * 60 * 60); /// Check for CLI updates, returning the latest version string if one is available. -pub(crate) async fn update_check(ctx: &icp::context::Context) -> Option { +pub(crate) async fn update_check(ctx: &icp_app::context::Context) -> Option { let update_check_setting = match ctx.dirs.settings() { Ok(dirs) => { - dirs.with_read(async |dirs| icp::settings::Settings::load_from(dirs).ok()) + dirs.with_read(async |dirs| icp_app::settings::Settings::load_from(dirs).ok()) .await .ok() .flatten() diff --git a/crates/icp-cli/src/main.rs b/crates/icp-cli/src/main.rs index 7db1bd947..ca3ad5c47 100644 --- a/crates/icp-cli/src/main.rs +++ b/crates/icp-cli/src/main.rs @@ -3,7 +3,8 @@ use std::sync::Arc; use anyhow::Error; use clap::{CommandFactory, Parser, ValueHint}; use commands::Command; -use icp::{directories::Access, prelude::*}; +use icp::prelude::*; +use icp_app::directories::Access; use tracing::{Instrument, debug, info, subscriber::set_global_default, trace_span}; use tracing_subscriber::{Registry, layer::SubscriberExt}; @@ -161,7 +162,7 @@ async fn run() -> Result<(), Error> { "Starting icp-cli" ); - let password_func: icp::identity::PasswordFunc = match cli.identity_password_file { + let password_func: icp_app::identity::PasswordFunc = match cli.identity_password_file { Some(path) => Arc::new(move || { icp::fs::read_to_string(&path) .map(|s| s.trim().to_string()) @@ -175,16 +176,16 @@ async fn run() -> Result<(), Error> { }), }; let pem_session_duration = { - let dirs = icp::directories::Directories::new()?; + let dirs = icp_app::directories::Directories::new()?; let settings_dirs = dirs.settings()?; let settings = settings_dirs - .with_read(async |dirs| icp::settings::Settings::load_from(dirs)) + .with_read(async |dirs| icp_app::settings::Settings::load_from(dirs)) .await??; settings .session_length .map(|m| std::time::Duration::from_secs((u64::from(m) + 2) * 60)) }; - let ctx = icp::context::initialize( + let ctx = icp_app::context::initialize( cli.project_root_override, cli.debug, password_func, @@ -202,7 +203,7 @@ async fn run() -> Result<(), Error> { let result = dispatch(&ctx, command).instrument(trace_span).await; if let Some(session) = telemetry_session { - session.finish(result.is_ok(), &ctx.host.telemetry_data); + session.finish(result.is_ok(), &ctx.telemetry_data); } // Show update nag after command output @@ -223,7 +224,7 @@ async fn run() -> Result<(), Error> { } /// Dispatch the command to its handler. -async fn dispatch(ctx: &icp::context::Context, command: Command) -> Result<(), Error> { +async fn dispatch(ctx: &icp_app::context::Context, command: Command) -> Result<(), Error> { match command { // Build Command::Build(args) => commands::build::exec(ctx, &args).await?, diff --git a/crates/icp-cli/src/options.rs b/crates/icp-cli/src/options.rs index cab7104ee..4feecdbcc 100644 --- a/crates/icp-cli/src/options.rs +++ b/crates/icp-cli/src/options.rs @@ -1,10 +1,11 @@ use clap::error::ErrorKind; use clap::{ArgGroup, ArgMatches, Args, FromArgMatches}; use clap_complete::ArgValueCandidates; -use icp::identity::IdentitySelection; +use icp::host::EnvironmentSelection; use icp::network::RootKeySpec; use icp::prelude::LOCAL; -use icp::{context::NetworkSelection, host::EnvironmentSelection}; +use icp_app::context::NetworkSelection; +use icp_app::identity::IdentitySelection; use url::Url; mod heading { diff --git a/crates/icp-cli/src/telemetry.rs b/crates/icp-cli/src/telemetry.rs index 39bbe21bd..6a36bb3e0 100644 --- a/crates/icp-cli/src/telemetry.rs +++ b/crates/icp-cli/src/telemetry.rs @@ -11,8 +11,8 @@ use std::{ use clap::parser::ValueSource; use icp::prelude::*; -use icp::settings::Settings; -use icp::telemetry_data::{IdentityStorageType, NetworkType, TelemetryData}; +use icp_app::settings::Settings; +use icp_app::telemetry_data::{IdentityStorageType, NetworkType, TelemetryData}; use rand::RngExt as _; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; @@ -161,7 +161,7 @@ impl TelemetrySession { /// Initialise a telemetry session unless telemetry is disabled. pub(crate) async fn setup( - ctx: &icp::context::Context, + ctx: &icp_app::context::Context, raw_args: &[String], clap_command: &clap::Command, ) -> Option { diff --git a/crates/icp-cli/tests/message_send_tests.rs b/crates/icp-cli/tests/message_send_tests.rs index 13de7114e..4bfeb3318 100644 --- a/crates/icp-cli/tests/message_send_tests.rs +++ b/crates/icp-cli/tests/message_send_tests.rs @@ -404,7 +404,7 @@ fn not_yet_valid_file_is_refused() { #[test] fn expired_file_is_refused() { use ic_agent::{Agent, identity::AnonymousIdentity}; - use icp::signed_message::{ + use icp_app::signed_message::{ CallType, Destination, Network, Request, SUBMISSION_WINDOW, SignedMessage, Summary, format_timestamp, }; @@ -434,8 +434,8 @@ fn expired_file_is_refused() { .expect("signing makes no request"); let message = SignedMessage { - format: icp::signed_message::FORMAT.to_string(), - version: icp::signed_message::VERSION, + format: icp_app::signed_message::FORMAT.to_string(), + version: icp_app::signed_message::VERSION, request: Request { call_type: CallType::Query, envelope: signed.signed_query, diff --git a/crates/icp/Cargo.toml b/crates/icp/Cargo.toml index 1ea3d7a77..21d9e68fd 100644 --- a/crates/icp/Cargo.toml +++ b/crates/icp/Cargo.toml @@ -5,61 +5,43 @@ edition = { workspace = true } license = { workspace = true } publish.workspace = true +[features] +# Exposes this crate's mocks and fixtures so downstream crates can test against +# the same seams. Off in a normal build, so none of it ships. +test-util = [] + [dependencies] -async-dropper = { workspace = true } async-trait = { workspace = true } -base64 = { workspace = true } -backoff = { workspace = true } bigdecimal = { workspace = true } -bip32 = { workspace = true } -bollard = { workspace = true } camino = { workspace = true } camino-tempfile = { workspace = true } candid = { workspace = true } -crypto-bigint = { workspace = true } candid_parser = { workspace = true } clap = { workspace = true, optional = true } -directories = { workspace = true } dunce = { workspace = true } -ed25519-consensus = { workspace = true } -elliptic-curve = { workspace = true } flate2 = { workspace = true } futures = { workspace = true } glob = { workspace = true } handlebars = { workspace = true } hex = { workspace = true } -hmac = { workspace = true } -hybrid-array = { workspace = true } ic-agent = { workspace = true } -ic-ed25519 = { workspace = true } -ic-identity-hsm = { workspace = true } ic-ledger-types = { workspace = true } ic-management-canister-types = { workspace = true } ic-utils = { workspace = true } +icrc-ledger-types = { workspace = true } icp-canister-interfaces = { workspace = true } icp-events = { workspace = true } icp-sync-plugin = { workspace = true } -icrc-ledger-types = { workspace = true } indexmap = { workspace = true } indoc = { workspace = true } itertools = { workspace = true } -k256 = { workspace = true } -keyring = { workspace = true } -notify = { workspace = true } num-bigint = { workspace = true } num-integer = { workspace = true } num-traits = { workspace = true } -p256 = { workspace = true } -phf = { workspace = true } pathdiff = { workspace = true } -pem = { workspace = true } -pkcs8 = { workspace = true } rand = { workspace = true } -reqwest = { workspace = true } schemars = { workspace = true } -scrypt = { workspace = true } semver = { workspace = true } -sec1 = { workspace = true } serde = { workspace = true } serde_cbor = { workspace = true } serde_json = { workspace = true } @@ -68,10 +50,8 @@ sha2 = { workspace = true } shellwords = { workspace = true } snafu = { workspace = true } strum = { workspace = true } -sysinfo = { workspace = true } tar = { workspace = true } time = { workspace = true } -tiny-bip39 = { workspace = true } # `io-std` is listed for `tokio::io::stdout`/`stderr`; feature unification with other # crates supplies it anyway, so dropping it would not fail the build. `rt-multi-thread` # is needed by `block_in_place` in `canister::sync::plugin` and arrives via the @@ -79,10 +59,7 @@ tiny-bip39 = { workspace = true } tokio = { workspace = true, features = ["sync", "macros", "rt", "time", "io-util", "io-std", "process", "signal"] } tracing = { workspace = true } url = { workspace = true } -uuid = { workspace = true } wasmparser = { workspace = true } -wslpath2 = { workspace = true } -zeroize = { workspace = true } [target.'cfg(windows)'.dependencies] winreg = { workspace = true } diff --git a/crates/icp/src/agent.rs b/crates/icp/src/agent.rs index 991075f37..c70a6ac8e 100644 --- a/crates/icp/src/agent.rs +++ b/crates/icp/src/agent.rs @@ -1,64 +1,17 @@ -use std::{error::Error, fmt, future::Future, sync::Arc, time::Duration}; +//! The agent an operation speaks through, without the means to make one. +//! +//! Building an agent takes an identity, a key store and a way to unlock it — +//! all of which belong to the surrounding application, not here. So this layer +//! names only what it needs: something it can ask for an agent when it has +//! something to say. + +use std::{error::Error, future::Future}; -use async_trait::async_trait; use futures::future::BoxFuture; -use ic_agent::{Agent, AgentError, Identity}; -use snafu::prelude::*; +use ic_agent::Agent; +use snafu::Snafu; use tokio::sync::OnceCell; -use crate::prelude::*; - -#[derive(Debug, Snafu)] -pub enum CreateAgentError { - #[snafu(display("failed to create agent"))] - Agent { source: AgentError }, -} - -/// How far ahead of now an agent dates the messages it expires, unless the -/// caller pins something else. -const DEFAULT_INGRESS_EXPIRY: Duration = Duration::from_secs(4 * MINUTE); - -#[async_trait] -pub trait Create: Sync + Send { - /// Builds an agent talking to `url` as `id`. - /// - /// `ingress_expiry` pins how far ahead of now the agent dates the messages it - /// derives an expiry for. Pass `None` for the default. Pass `Some` only when - /// the expiry is itself part of the output — signing a message here for - /// another machine to submit, where the call envelope and the pre-signed - /// `request_status` that accompanies it have to land in the same submission - /// window. A pinned expiry is used verbatim, so the - /// `ICP_CLI_TEST_ADVANCE_TIME_MS` clock offset applies to the default only. - async fn create( - &self, - id: Arc, - url: &str, - ingress_expiry: Option, - ) -> Result; -} - -pub struct Creator; - -#[async_trait] -impl Create for Creator { - async fn create( - &self, - id: Arc, - url: &str, - ingress_expiry: Option, - ) -> Result { - let ingress_expiry = - ingress_expiry.unwrap_or_else(|| DEFAULT_INGRESS_EXPIRY + test_time_advance()); - - let b = Agent::builder() - .with_url(url) - .with_arc_identity(id) - .with_ingress_expiry(ingress_expiry); - - Ok(b.build().context(AgentSnafu)?) - } -} - /// An [`Agent`] created on first use. /// /// Creating an agent unlocks an identity — a password prompt, for an encrypted @@ -87,7 +40,7 @@ impl<'a> LazyAgent<'a> { cell: OnceCell::new(), create: Box::new(move || { let creating = create(); - Box::pin(async move { creating.await.map_err(|e| LazyAgentError(Box::new(e))) }) + Box::pin(async move { creating.await.map_err(LazyAgentError::new) }) }), } } @@ -98,36 +51,24 @@ impl<'a> LazyAgent<'a> { } } -/// Whatever went wrong in a [`LazyAgent`]'s creation function. +/// An agent could not be created. /// -/// Type-erased, and hand-written rather than a Snafu variant, because how an -/// identity is resolved and unlocked belongs to the caller: an operation holding -/// a `LazyAgent` cannot name that error, and does nothing with it but report it. -/// So this adds no message of its own — display and source both pass straight -/// through, as `snafu(transparent)` would. -#[derive(Debug)] -pub struct LazyAgentError(Box); - -impl fmt::Display for LazyAgentError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl Error for LazyAgentError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - self.0.source() - } +/// What that took is the caller's business: resolving an identity, unlocking a +/// key, reaching a network for its root key. This layer knows only that it can +/// fail and that whatever went wrong is what the user needs to be told, so the +/// cause is carried whole and displayed as itself rather than being restated +/// here. +#[derive(Debug, Snafu)] +#[snafu(transparent)] +pub struct LazyAgentError { + pub source: Box, } -/// How far a test has advanced the replica's clock past this machine's, so the -/// default ingress expiry stays ahead of replica time. -fn test_time_advance() -> Duration { - match std::env::var("ICP_CLI_TEST_ADVANCE_TIME_MS") { - Ok(ms) => Duration::from_millis( - ms.parse::() - .expect("ICP_CLI_TEST_ADVANCE_TIME_MS must be set to an int"), - ), - Err(_) => Duration::ZERO, +impl LazyAgentError { + /// Wraps a creation function's own error for the boundary. + pub fn new(source: impl Error + Send + Sync + 'static) -> Self { + Self { + source: Box::new(source), + } } } diff --git a/crates/icp/src/canister/build/mod.rs b/crates/icp/src/canister/build/mod.rs index 7324e20f6..3aafef453 100644 --- a/crates/icp/src/canister/build/mod.rs +++ b/crates/icp/src/canister/build/mod.rs @@ -3,8 +3,10 @@ use async_trait::async_trait; use icp_events::StepReporter; use snafu::prelude::*; +use std::sync::Arc; + +use crate::canister::wasm; use crate::manifest::canister::BuildStep; -use crate::package::PackageCache; use crate::prelude::*; mod prebuilt; @@ -31,11 +33,20 @@ pub trait Build: Sync + Send { step: &BuildStep, params: &Params, reporter: &StepReporter, - pkg_cache: &PackageCache, ) -> Result<(), BuildError>; } -pub struct Builder; +/// Runs each build step where it has to be run: a script step in a subprocess, +/// a pre-built step by asking [`wasm::Fetch`] for the module. +pub struct Builder { + wasm: Arc, +} + +impl Builder { + pub fn new(wasm: Arc) -> Self { + Self { wasm } + } +} #[async_trait] impl Build for Builder { @@ -44,23 +55,22 @@ impl Build for Builder { step: &BuildStep, params: &Params, reporter: &StepReporter, - pkg_cache: &PackageCache, ) -> Result<(), BuildError> { match step { BuildStep::Prebuilt(adapter) => { - Ok(prebuilt::build(adapter, params, reporter, pkg_cache).await?) + Ok(prebuilt::build(adapter, params, reporter, self.wasm.as_ref()).await?) } BuildStep::Script(adapter) => Ok(script::build(adapter, params, reporter).await?), } } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] /// Unimplemented mock implementation of `Build`. /// All methods panic with `unimplemented!()` when called. pub struct UnimplementedMockBuilder; -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] #[async_trait] impl Build for UnimplementedMockBuilder { async fn build( @@ -68,7 +78,6 @@ impl Build for UnimplementedMockBuilder { _step: &BuildStep, _params: &Params, _reporter: &StepReporter, - _pkg_cache: &PackageCache, ) -> Result<(), BuildError> { unimplemented!("UnimplementedMockBuilder::build") } diff --git a/crates/icp/src/canister/build/prebuilt.rs b/crates/icp/src/canister/build/prebuilt.rs index d0482756f..c2dbb22bf 100644 --- a/crates/icp/src/canister/build/prebuilt.rs +++ b/crates/icp/src/canister/build/prebuilt.rs @@ -1,14 +1,14 @@ use icp_events::StepReporter; use snafu::prelude::*; -use crate::{canister::wasm, fs, manifest::adapter::prebuilt::Adapter, package::PackageCache}; +use crate::{canister::wasm, fs, manifest::adapter::prebuilt::Adapter}; use super::Params; #[derive(Debug, Snafu)] pub enum PrebuiltError { #[snafu(transparent)] - Wasm { source: wasm::WasmError }, + Wasm { source: wasm::FetchError }, #[snafu(display("failed to copy wasm to output file"))] CopyFile { source: crate::fs::CopyError }, @@ -18,16 +18,16 @@ pub(super) async fn build( adapter: &Adapter, params: &Params, reporter: &StepReporter, - pkg_cache: &PackageCache, + wasm: &dyn wasm::Fetch, ) -> Result<(), PrebuiltError> { - let src = wasm::resolve( - &adapter.source, - ¶ms.path, - adapter.sha256.as_deref(), - reporter, - pkg_cache, - ) - .await?; + let src = wasm + .wasm( + &adapter.source, + ¶ms.path, + adapter.sha256.as_deref(), + reporter, + ) + .await?; reporter.info(format!("Writing WASM file: {}", params.output)); fs::copy(&src, ¶ms.output).context(CopyFileSnafu)?; diff --git a/crates/icp/src/canister/recipe/mod.rs b/crates/icp/src/canister/recipe/mod.rs index edb92caf3..5119c12b0 100644 --- a/crates/icp/src/canister/recipe/mod.rs +++ b/crates/icp/src/canister/recipe/mod.rs @@ -1,59 +1,74 @@ //! Recipe resolution, split into two stages. //! -//! [`fetch`] retrieves a recipe's Handlebars template — reading a local file, or -//! downloading a remote URL or registry recipe — and returns the raw template -//! text. [`render`] turns that text into concrete build/sync steps. The first -//! stage does I/O and nothing else; the second is a pure function. +//! [`Resolve`] retrieves a recipe's Handlebars template — reading a local file, +//! or downloading a remote URL or registry recipe — and returns the raw +//! template text. [`render`] turns that text into concrete build/sync steps. +//! The first stage does I/O and nothing else; the second is a pure function. //! -//! The [`Resolve`] seam therefore covers only the fetching half, so a caller that -//! already has a template (or must not touch the network) can render without -//! going through a resolver at all. +//! The seam therefore covers only the fetching half, so a caller that already +//! has a template (or must not touch the network) can render without going +//! through a resolver at all. //! //! Caching a download is a third step, because whether a template is worth -//! keeping is not known until it renders. A download that carried a `sha256` is -//! cached during the fetch — the checksum already proves the bytes are the ones -//! that were asked for. An *unpinned* download is held back as a -//! [`PendingCache`] and only committed by the caller once rendering succeeds, so -//! that one bad remote response cannot become sticky in the cache. The full -//! sequence is therefore fetch → render → [`Resolve::commit`]. +//! keeping is not known until it renders: one bad remote response must not +//! become sticky in the cache. A resolver that wants to cache says so by +//! returning [`Fetched::deferred`], and the caller calls +//! [`Resolve::commit`] once rendering has succeeded. What is then written, and +//! where, is the resolver's own business — nothing about a cache crosses this +//! trait. use async_trait::async_trait; -use snafu::prelude::*; +use snafu::Snafu; use crate::manifest::recipe::Recipe; -pub mod fetch; pub mod render; -pub use fetch::{Fetched, PendingCache}; pub use render::{RecipeContext, RenderRecipeError, render_recipe}; -/// Retrieves the recipe templates a project references. +/// A recipe template, as retrieved by a [`Resolve`]. +pub struct Fetched { + /// Raw Handlebars template source. + pub template: String, + + /// Whether the resolver is holding work back until the template is known + /// to render. A resolver that caches nothing leaves this `false` and is + /// never asked to commit. + pub deferred: bool, +} + +/// A recipe template could not be retrieved or could not be cached. /// -/// Only *fetching* is behind this trait: rendering a fetched template into build -/// and sync steps is [`render_recipe`], which needs no I/O and so needs no seam. +/// Fetching one may mean an HTTP request and a write to a cache outside the +/// project. This layer knows only that it can fail, so the cause is carried +/// whole and displayed as itself. +#[derive(Debug, Snafu)] +#[snafu(transparent)] +pub struct ResolveError { + pub source: Box, +} + +impl ResolveError { + /// Wraps an implementation's own error for the trait boundary. + pub fn new(source: impl std::error::Error + Send + Sync + 'static) -> Self { + Self { + source: Box::new(source), + } + } +} + +/// Retrieves the recipe templates a project references. #[async_trait] pub trait Resolve: Sync + Send { - /// Fetch the Handlebars template for `recipe`, returning its raw source and - /// any cache write held back until the template is known to render. + /// Fetch the Handlebars template for `recipe`. async fn resolve(&self, recipe: &Recipe) -> Result; - /// Write a held-back download to the cache, now that it has rendered. + /// Let the resolver finish whatever it deferred, now that the template it + /// returned has rendered. /// - /// Defaults to doing nothing: only [`fetch::RecipeFetcher`] caches, and only - /// it can construct the [`PendingCache`] that reaches this method, so a - /// resolver that never defers a write never has one to commit. - async fn commit(&self, pending: PendingCache) -> Result<(), ResolveError> { - let _ = pending; + /// Defaults to doing nothing, for a resolver that never defers. + async fn commit(&self, recipe: &Recipe, fetched: &Fetched) -> Result<(), ResolveError> { + let _ = (recipe, fetched); Ok(()) } } - -#[derive(Debug, Snafu)] -pub enum ResolveError { - #[snafu(display("failed to fetch recipe template"))] - Fetch { source: fetch::RecipeFetchError }, - - #[snafu(display("failed to cache recipe template"))] - Commit { source: fetch::RecipeFetchError }, -} diff --git a/crates/icp/src/canister/sync/mod.rs b/crates/icp/src/canister/sync/mod.rs index b5b033321..66f0e314f 100644 --- a/crates/icp/src/canister/sync/mod.rs +++ b/crates/icp/src/canister/sync/mod.rs @@ -7,9 +7,9 @@ use ic_agent::Agent; use icp_events::StepReporter; use snafu::prelude::*; +use crate::canister::wasm; use crate::manifest::canister::SyncStep; use crate::network::NetworkUrls; -use crate::package::PackageCache; use crate::prelude::*; mod plugin; @@ -60,7 +60,6 @@ pub trait Synchronize: Sync + Send { params: &Params, agent: &Agent, reporter: &StepReporter, - pkg_cache: &PackageCache, ) -> Result, SynchronizeError>; } @@ -70,16 +69,17 @@ pub trait Synchronize: Sync + Send { /// everywhere. pub struct Syncer { scripts: Arc, + wasm: Arc, } impl Syncer { /// A syncer that runs script steps as host subprocesses. - pub fn host() -> Self { - Self::new(Arc::new(HostScripts)) + pub fn host(wasm: Arc) -> Self { + Self::new(Arc::new(HostScripts), wasm) } - pub fn new(scripts: Arc) -> Self { - Self { scripts } + pub fn new(scripts: Arc, wasm: Arc) -> Self { + Self { scripts, wasm } } } @@ -91,7 +91,6 @@ impl Synchronize for Syncer { params: &Params, agent: &Agent, reporter: &StepReporter, - pkg_cache: &PackageCache, ) -> Result, SynchronizeError> { match step { SyncStep::Script(adapter) => Ok(self @@ -105,19 +104,19 @@ impl Synchronize for Syncer { ¶ms.environment, params.proxy, reporter, - pkg_cache, + self.wasm.as_ref(), ) .await?), } } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] /// Unimplemented mock implementation of `Synchronize`. /// All methods panic with `unimplemented!()` when called. pub struct UnimplementedMockSyncer; -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] #[async_trait] impl Synchronize for UnimplementedMockSyncer { async fn sync( @@ -126,7 +125,6 @@ impl Synchronize for UnimplementedMockSyncer { _params: &Params, _agent: &Agent, _reporter: &StepReporter, - _pkg_cache: &PackageCache, ) -> Result, SynchronizeError> { unimplemented!("UnimplementedMockSyncer::sync") } @@ -172,7 +170,7 @@ mod tests { #[tokio::test] async fn script_steps_are_dispatched_to_the_injected_runner() { let scripts = Arc::new(RecordingScripts::default()); - let syncer = Syncer::new(scripts.clone()); + let syncer = Syncer::new(scripts.clone(), Arc::new(wasm::UnimplementedMockFetch)); let cid = Principal::from_slice(&[7; 4]); let params = Params { @@ -196,17 +194,8 @@ mod tests { command: CommandField::Command("./deploy.sh".to_owned()), }); - let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); - let pkg_cache = PackageCache::new(tmp.path().to_owned()).unwrap(); - let retained = syncer - .sync( - &step, - ¶ms, - &dummy_agent(), - &StepReporter::null(), - &pkg_cache, - ) + .sync(&step, ¶ms, &dummy_agent(), &StepReporter::null()) .await .expect("script step should dispatch"); assert!(retained.is_empty()); diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 2a2a0b2d6..256bfd733 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -13,7 +13,6 @@ use snafu::prelude::*; use crate::{ canister::wasm, manifest::adapter::plugin::{Adapter, NamedPaths}, - package::PackageCache, }; use super::Params; @@ -34,7 +33,7 @@ fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { #[derive(Debug, Snafu)] pub enum PluginError { #[snafu(transparent)] - Wasm { source: wasm::WasmError }, + Wasm { source: wasm::FetchError }, #[snafu(display("failed to get identity principal: {err}"))] GetIdentityPrincipal { err: String }, @@ -89,7 +88,7 @@ pub(super) async fn sync( environment: &str, proxy: Option, reporter: &StepReporter, - pkg_cache: &PackageCache, + wasm_fetch: &dyn wasm::Fetch, ) -> Result, PluginError> { // 0. Resolve the compute-time limit up front so a malformed // ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS fails fast — before downloading the @@ -100,14 +99,14 @@ pub(super) async fn sync( // - Local: sha256 is verified if present, then the original path is returned. // - Remote: downloaded to cache (sha256 required, enforced at parse time) and the // stable cache path is returned — no temp file needed. - let wasm_path = wasm::resolve( - &adapter.source, - ¶ms.path, - adapter.sha256.as_deref(), - reporter, - pkg_cache, - ) - .await?; + let wasm_path = wasm_fetch + .wasm( + &adapter.source, + ¶ms.path, + adapter.sha256.as_deref(), + reporter, + ) + .await?; // 2. Collect inputs as manifest strings. `run_plugin` opens the declared // paths itself — preopening or reading each by what is on disk, anchored diff --git a/crates/icp/src/canister/wasm.rs b/crates/icp/src/canister/wasm.rs index c77549dba..c45eef18f 100644 --- a/crates/icp/src/canister/wasm.rs +++ b/crates/icp/src/canister/wasm.rs @@ -1,138 +1,63 @@ -use camino::{Utf8Path, Utf8PathBuf}; +//! Getting hold of a wasm module the project points at but does not contain. + +use camino::Utf8Path; use icp_events::StepReporter; -use reqwest::{Client, Method, Request}; -use sha2::{Digest, Sha256}; -use snafu::prelude::*; -use url::Url; +use snafu::Snafu; -use crate::{ - fs::read, - manifest::adapter::prebuilt::SourceField, - package::{PackageCache, cache_wasm}, -}; +use crate::manifest::adapter::prebuilt::SourceField; +use crate::prelude::*; +/// A wasm module could not be produced. +/// +/// A manifest may name a module by URL, so resolving one can mean an HTTP +/// request and a write to a cache outside the project. This layer knows only +/// that it can fail, so the cause is carried whole and displayed as itself. #[derive(Debug, Snafu)] -pub enum WasmError { - #[snafu(display("failed to read wasm file at '{path}'"))] - ReadLocal { - source: crate::fs::IoError, - path: Utf8PathBuf, - }, - - #[snafu(display("failed to parse wasm url"))] - ParseUrl { source: url::ParseError }, - - #[snafu(display("failed to fetch wasm file"))] - HttpRequest { source: reqwest::Error }, - - #[snafu(display("http request failed: {status}"))] - HttpStatus { status: reqwest::StatusCode }, - - #[snafu(display("failed to read http response"))] - HttpResponse { source: reqwest::Error }, - - #[snafu(display("checksum mismatch, expected: {expected}, actual: {actual}"))] - ChecksumMismatch { expected: String, actual: String }, - - #[snafu(display("failed to cache wasm file"))] - CacheFile { source: crate::fs::IoError }, - - #[snafu(display("failed to acquire lock on package cache"))] - LockCache { source: crate::fs::lock::LockError }, +#[snafu(transparent)] +pub struct FetchError { + pub source: Box, } -/// Resolve a wasm source to a local filesystem path, optionally verifying the sha256 checksum. -/// -/// - Local: verifies sha256 if provided, returns the local path. -/// - Remote with sha256: checks the cache first; downloads, verifies, and caches on miss. -/// - Remote without sha256: always downloads, computes sha256, caches by the computed sha256. -pub async fn resolve( - source: &SourceField, - base_dir: &Utf8Path, - sha256: Option<&str>, - reporter: &StepReporter, - pkg_cache: &PackageCache, -) -> Result { - match source { - SourceField::Local(s) => { - let path = base_dir.join(&s.path); - if let Some(expected) = sha256 { - reporter.info(format!("Reading wasm: {}", s.path)); - let bytes = read(&path).context(ReadLocalSnafu { - path: s.path.clone(), - })?; - reporter.info("Verifying checksum"); - let actual = hex::encode(Sha256::digest(&bytes)); - ensure!( - actual == expected, - ChecksumMismatchSnafu { - expected: expected.to_owned(), - actual, - } - ); - } - Ok(path) +impl FetchError { + /// Wraps an implementation's own error for the trait boundary. + pub fn new(source: impl std::error::Error + Send + Sync + 'static) -> Self { + Self { + source: Box::new(source), } - SourceField::Remote(s) => { - // Pre-download cache check is only possible when sha256 is known. - if let Some(expected) = sha256 { - let cached = pkg_cache - .with_read(async |r| { - let wasm_cache = r.wasm_sha(expected); - let path = wasm_cache.wasm(); - if path.exists() { - _ = crate::fs::write(&wasm_cache.atime(), b""); - Some(path) - } else { - None - } - }) - .await - .context(LockCacheSnafu)?; - if let Some(path) = cached { - reporter.info("Using cached file"); - return Ok(path); - } - } - - let url = Url::parse(&s.url).context(ParseUrlSnafu)?; - reporter.info(format!("Fetching wasm: {url}")); - let resp = Client::new() - .execute(Request::new(Method::GET, url)) - .await - .context(HttpRequestSnafu)?; - let status = resp.status(); - if !status.is_success() { - return HttpStatusSnafu { status }.fail(); - } - let bytes = resp.bytes().await.context(HttpResponseSnafu)?.to_vec(); - - // Use provided sha256 as cache key (after verifying), or compute from bytes. - let cache_sha = match sha256 { - Some(expected) => { - reporter.info("Verifying checksum"); - let actual = hex::encode(Sha256::digest(&bytes)); - ensure!( - actual == expected, - ChecksumMismatchSnafu { - expected: expected.to_owned(), - actual, - } - ); - actual - } - None => hex::encode(Sha256::digest(&bytes)), - }; + } +} - pkg_cache - .with_write(async |w| cache_wasm(w, &cache_sha, &bytes).context(CacheFileSnafu)) - .await - .context(LockCacheSnafu)??; +/// Where a build or sync step gets the wasm module it was told to use. +/// +/// Asked for rather than done here: fetching over HTTP and writing to a cache +/// outside the project are not available to every caller of this crate. +#[async_trait::async_trait] +pub trait Fetch: Send + Sync { + /// Resolve a wasm source to a local file, verifying `sha256` when one is + /// given. + async fn wasm( + &self, + source: &SourceField, + base_dir: &Utf8Path, + sha256: Option<&str>, + reporter: &StepReporter, + ) -> Result; +} - pkg_cache - .with_read(async |r| r.wasm_sha(&cache_sha).wasm()) - .await - .context(LockCacheSnafu) - } +#[cfg(any(test, feature = "test-util"))] +/// A [`Fetch`] for tests on paths that never reach a wasm source. +pub struct UnimplementedMockFetch; + +#[cfg(any(test, feature = "test-util"))] +#[async_trait::async_trait] +impl Fetch for UnimplementedMockFetch { + async fn wasm( + &self, + _source: &SourceField, + _base_dir: &Utf8Path, + _sha256: Option<&str>, + _reporter: &StepReporter, + ) -> Result { + unimplemented!("UnimplementedMockFetch::wasm") } } diff --git a/crates/icp/src/host.rs b/crates/icp/src/host.rs index 9cc60ca84..a8ac7633f 100644 --- a/crates/icp/src/host.rs +++ b/crates/icp/src/host.rs @@ -21,7 +21,6 @@ use crate::{ network::{Configuration as NetworkConfiguration, FriendlyDomains}, prelude::*, store_id::{IdMapping, LookupIdError}, - telemetry_data::NetworkType, }; /// Selection type for environments @@ -69,18 +68,40 @@ pub struct Host { /// Canister synchronizer pub syncer: Arc, + /// Source of wasm modules a manifest names but the project does not contain + pub wasm: Arc, + /// Network resolution: endpoints, root keys, friendly domains pub network: Arc, - /// Telemetry data collected during command execution. - // TODO: telemetry is app-global, not project-scoped. Once the app layer is - // its own crate, it should derive these facts from the loaded project - // itself rather than having project loading write into a bag it owns. - pub telemetry_data: Arc, + /// Where to report what resolution turned up. See [`Observe`]. + pub observer: Arc, +} + +/// Somewhere for the surrounding application to notice what resolution turned +/// up. +/// +/// Which project is loaded, and which environment was picked out of it, are +/// facts an application wants — for telemetry, for a status line — and they are +/// established in here, part-way through resolving something else, not at the +/// call site. Rather than let an app-scoped collector be written to from this +/// layer, the facts are handed over and what becomes of them is not this +/// layer's concern. +pub trait Observe: Send + Sync { + /// An environment was resolved out of a loaded project. + /// + /// Called on every resolution, not just the first, so implementations must + /// tolerate being told the same thing repeatedly. + fn environment_resolved(&self, _project: &crate::Project, _environment: &crate::Environment) {} } +/// The [`Observe`] for a caller that does not care. +pub struct Ignore; + +impl Observe for Ignore {} + impl Host { - #[cfg(test)] + #[cfg(any(test, feature = "test-util"))] /// A host whose every seam is a mock, for tests that only exercise the /// resolution methods below. pub fn mocked() -> Self { @@ -90,8 +111,9 @@ impl Host { artifacts: Arc::new(crate::store_artifact::MockInMemoryArtifactStore::new()), builder: Arc::new(crate::canister::build::UnimplementedMockBuilder), syncer: Arc::new(crate::canister::sync::UnimplementedMockSyncer), + wasm: Arc::new(crate::canister::wasm::UnimplementedMockFetch), network: Arc::new(crate::network::MockNetworkAccessor::new()), - telemetry_data: Arc::new(crate::telemetry_data::TelemetryData::default()), + observer: Arc::new(Ignore), } } @@ -128,12 +150,7 @@ impl Host { .fail(); } - let network_type = match &env.network.configuration { - NetworkConfiguration::Managed { .. } => NetworkType::Managed, - NetworkConfiguration::Connected { .. } => NetworkType::Connected, - }; - self.telemetry_data.set_network_type(network_type); - self.telemetry_data.set_project(&p); + self.observer.environment_resolved(&p, env); Ok(env.clone()) } diff --git a/crates/icp/src/lib.rs b/crates/icp/src/lib.rs index a788e3a50..cab2d97d7 100644 --- a/crates/icp/src/lib.rs +++ b/crates/icp/src/lib.rs @@ -26,28 +26,24 @@ use crate::{ pub mod agent; pub mod canister; -pub mod context; -pub mod directories; pub mod fs; pub mod host; -pub mod identity; pub mod manifest; pub mod network; pub mod operations; -pub mod package; pub mod parsers; pub mod prelude; pub mod project; -pub mod settings; pub mod signal; -pub mod signed_message; pub mod store_artifact; pub mod store_id; -pub mod telemetry_data; -const ICP_BASE: &str = ".icp"; -const CACHE_DIR: &str = "cache"; -const DATA_DIR: &str = "data"; +/// The per-project state directory. `cache` holds what can be thrown away and +/// rebuilt (ids for managed networks, build artifacts); `data` holds what +/// cannot (ids for connected networks). +pub const ICP_BASE: &str = ".icp"; +pub const CACHE_DIR: &str = "cache"; +pub const DATA_DIR: &str = "data"; /// Resolved canister arguments, with any file references already loaded. #[derive(Clone, Debug, PartialEq, Serialize)] @@ -346,14 +342,14 @@ impl ProjectLoad for Lazy { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] /// Mock project loader for testing. /// Returns a pre-configured `Project` when `load()` is called. pub struct MockProjectLoader { project: Project, } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] impl MockProjectLoader { /// Creates a new mock project loader with the given project. pub fn new(project: Project) -> Self { @@ -671,7 +667,7 @@ impl MockProjectLoader { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] #[async_trait] impl ProjectLoad for MockProjectLoader { async fn load(&self) -> Result { @@ -683,12 +679,12 @@ impl ProjectLoad for MockProjectLoader { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] /// Mock project loader that always fails with a Locate error. /// Useful for testing scenarios where no project exists. pub struct NoProjectLoader; -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] #[async_trait] impl ProjectLoad for NoProjectLoader { async fn load(&self) -> Result { @@ -747,7 +743,7 @@ mod tests { path: dummy.wasm "#} .to_owned(), - pending_cache: None, + deferred: false, }) } } diff --git a/crates/icp/src/manifest/mod.rs b/crates/icp/src/manifest/mod.rs index 71668a23c..10d54c27f 100644 --- a/crates/icp/src/manifest/mod.rs +++ b/crates/icp/src/manifest/mod.rs @@ -14,7 +14,7 @@ pub(crate) mod dependency; pub(crate) mod environment; pub(crate) mod network; pub(crate) mod project; -pub(crate) mod recipe; +pub mod recipe; pub(crate) mod serde_helpers; pub use { diff --git a/crates/icp/src/network/access.rs b/crates/icp/src/network/access.rs index 0a0309f9d..0f3aefbc5 100644 --- a/crates/icp/src/network/access.rs +++ b/crates/icp/src/network/access.rs @@ -1,21 +1,12 @@ -use std::sync::Arc; +//! What a network is reached at, and how much its answers can be trusted. +//! +//! These are the values [`Access`](super::Access) hands back. *Producing* them +//! — reading a descriptor some launcher wrote, fetching a root key over HTTP — +//! is the implementation's business, not this crate's. -use ic_agent::{AgentError, identity::AnonymousIdentity}; use serde::Serialize; -use snafu::{OptionExt, ResultExt, Snafu}; use url::Url; -use crate::{ - agent::{Create, CreateAgentError}, - context::IC_ROOT_KEY, - manifest::network::RootKeySpec, - network::{ - Connected, NetworkDirectory, config::NetworkDescriptorModel, - directory::LoadNetworkFileError, - }, - prelude::*, -}; - /// Where a network's root key came from. Used for display so users can tell a /// trusted/pinned key apart from one that was fetched trust-on-first-use. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] @@ -59,156 +50,3 @@ pub struct NetworkAccess { /// If true, use friendly canister names with the gateway url pub use_friendly_domains: bool, } - -#[derive(Debug, Snafu)] -pub enum GetNetworkAccessError { - #[snafu(display("failed to load port {port} descriptor"))] - LoadPortDescriptor { - port: u16, - source: LoadNetworkFileError, - }, - - #[snafu(display("the {network} network for this project is not running"))] - NetworkNotRunning { network: String }, - - #[snafu(display( - "port {port} is already in use by the {network} network of another project at {project_dir}" - ))] - NetworkRunningOtherProject { - network: String, - port: u16, - project_dir: PathBuf, - }, - - #[snafu(display("no descriptor found for port {port}"))] - NoPortDescriptor { port: u16 }, - - #[snafu(display("failed to load network descriptor"))] - LoadNetworkDescriptor { source: LoadNetworkFileError }, - - #[snafu(display("failed to create agent to fetch root key from {url}"))] - CreateBootstrapAgent { - url: Url, - #[snafu(source(from(CreateAgentError, Box::new)))] - source: Box, - }, - - #[snafu(display("failed to fetch root key from {url}"))] - FetchRootKey { - url: Url, - #[snafu(source(from(AgentError, Box::new)))] - source: Box, - }, -} - -pub async fn get_managed_network_access( - nd: NetworkDirectory, -) -> Result { - let (desc, gateway_url) = managed_network_gateway(nd).await?; - Ok(NetworkAccess { - root_key: desc.root_key, - root_key_source: RootKeySource::Managed, - api_url: gateway_url.clone(), - http_gateway_url: Some(gateway_url), - use_friendly_domains: desc.use_friendly_domains, - }) -} - -/// The URLs a running managed network is reached at. Its gateway serves the API -/// as well, so both URLs are the same one. -pub async fn get_managed_network_urls( - nd: NetworkDirectory, -) -> Result { - let (_, gateway_url) = managed_network_gateway(nd).await?; - Ok(NetworkUrls { - api_url: gateway_url.clone(), - http_gateway_url: Some(gateway_url), - }) -} - -/// A running managed network's descriptor and the URL its gateway is reachable -/// at. A network that is not running has no descriptor, and one whose fixed port -/// has since been taken by another project's network is not the network the -/// descriptor describes — both are errors rather than a URL nothing answers on. -async fn managed_network_gateway( - nd: NetworkDirectory, -) -> Result<(NetworkDescriptorModel, Url), GetNetworkAccessError> { - // Load network descriptor - let desc = nd - .load_network_descriptor() - .await - .context(LoadNetworkDescriptorSnafu)? - .ok_or(GetNetworkAccessError::NetworkNotRunning { - network: nd.network_name.to_owned(), - })?; - - // Specify port - let port = desc.gateway.port; - - // Apply gateway configuration - if desc.gateway.fixed { - let pdesc = nd - .load_port_descriptor(port) - .await - .context(LoadPortDescriptorSnafu { port })? - .context(NoPortDescriptorSnafu { port })?; - - if desc.id != pdesc.id { - return NetworkRunningOtherProjectSnafu { - network: pdesc.network, - port: pdesc.gateway.port, - project_dir: pdesc.project_dir, - } - .fail(); - } - } - let http_gateway_url = Url::parse(&format!("http://{}:{port}", desc.gateway.host)).unwrap(); - Ok((desc, http_gateway_url)) -} - -pub async fn get_connected_network_access( - connected: &Connected, - agent: &Arc, -) -> Result { - let (root_key, root_key_source) = match &connected.root_key { - RootKeySpec::Mainnet => (IC_ROOT_KEY.to_vec(), RootKeySource::Mainnet), - RootKeySpec::Explicit(bytes) => (bytes.clone(), RootKeySource::Configured), - RootKeySpec::Fetch => { - let root_key = fetch_root_key(agent, &connected.api_url).await?; - (root_key, RootKeySource::Fetched) - } - }; - - Ok(NetworkAccess { - root_key, - root_key_source, - api_url: connected.api_url.clone(), - http_gateway_url: connected.http_gateway_url.clone(), - use_friendly_domains: false, - }) -} - -/// Fetch a network's root key trust-on-first-use. This does *not* verify the -/// key's provenance, so we warn the user that responses cannot be trusted the -/// way a pinned key allows. -async fn fetch_root_key( - agent: &Arc, - api_url: &Url, -) -> Result, GetNetworkAccessError> { - tracing::warn!( - "fetching the root key from {api_url}; its provenance is not verified (trust-on-first-use)" - ); - let bootstrap = agent - .create(Arc::new(AnonymousIdentity), api_url.as_str(), None) - .await - .context(CreateBootstrapAgentSnafu { - url: api_url.clone(), - })?; - bootstrap - .fetch_root_key() - .await - .context(FetchRootKeySnafu { - url: api_url.clone(), - })?; - Ok(bootstrap.read_root_key()) -} diff --git a/crates/icp/src/network/mod.rs b/crates/icp/src/network/mod.rs index d30810946..ca38535b9 100644 --- a/crates/icp/src/network/mod.rs +++ b/crates/icp/src/network/mod.rs @@ -1,5 +1,3 @@ -use std::{collections::BTreeMap, sync::Arc}; - use async_trait::async_trait; use candid::Principal; use schemars::JsonSchema; @@ -7,31 +5,19 @@ use serde::{Deserialize, Deserializer, Serialize}; use snafu::prelude::*; pub use crate::manifest::network::RootKeySpec; -pub use access::{NetworkUrls, RootKeySource}; -pub use directory::{LoadPidError, NetworkDirectory, SavePidError}; -pub use managed::run::{RunNetworkError, run_network}; +pub use access::{NetworkAccess, NetworkUrls, RootKeySource}; use strum::EnumString; use url::Url; use crate::{ - CACHE_DIR, ICP_BASE, Network, - manifest::{ - ProjectRootLocate, ProjectRootLocateError, - network::{Connected as ManifestConnected, Endpoints, Gateway as ManifestGateway, Mode}, - }, - network::access::{ - GetNetworkAccessError, NetworkAccess, get_connected_network_access, - get_managed_network_access, get_managed_network_urls, + Network, + manifest::network::{ + Connected as ManifestConnected, Endpoints, Gateway as ManifestGateway, Mode, }, - prelude::*, project::DEFAULT_LOCAL_NETWORK_PORT, }; pub mod access; -pub mod config; -pub mod custom_domains; -pub mod directory; -pub mod managed; #[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] pub enum Port { @@ -336,13 +322,26 @@ impl From for Configuration { } } +/// A network could not be reached, or could not be described. +/// +/// What that took is the implementation's business: locating the project, +/// reading a descriptor some launcher wrote, fetching a root key over HTTP. +/// This layer knows only that it can fail and that whatever went wrong is what +/// the user needs to be told, so the cause is carried whole and displayed as +/// itself rather than being restated here. #[derive(Debug, Snafu)] -pub enum AccessError { - #[snafu(display("failed to find project root"))] - ProjectRootLocate { source: ProjectRootLocateError }, +#[snafu(transparent)] +pub struct AccessError { + pub source: Box, +} - #[snafu(transparent)] - GetNetworkAccess { source: GetNetworkAccessError }, +impl AccessError { + /// Wraps an implementation's own error for the trait boundary. + pub fn new(source: impl std::error::Error + Send + Sync + 'static) -> Self { + Self { + source: Box::new(source), + } + } } /// One environment's friendly-name mappings, as collected from the project. @@ -367,7 +366,8 @@ pub type CollectFriendlyDomains<'a> = dyn Fn(&str) -> Vec + Sen #[async_trait] pub trait Access: Sync + Send { - fn get_network_directory(&self, network: &Network) -> Result; + /// The network's endpoints together with the trust material needed to + /// verify what it says. async fn access(&self, network: &Network) -> Result; /// The network's URLs alone. Unlike [`Access::access`] this resolves no root @@ -393,118 +393,24 @@ pub trait Access: Sync + Send { ); } -pub struct Accessor { - // Project root - pub project_root_locate: Arc, - - // Port descriptors dir - pub descriptors: PathBuf, - - // Used to build a bootstrap agent when a connected network fetches its root key - pub agent: Arc, -} - -#[async_trait] -impl Access for Accessor { - /// The network directory is located at `/.icp/cache/networks/`. - fn get_network_directory(&self, network: &Network) -> Result { - let dir = self - .project_root_locate - .locate() - .context(ProjectRootLocateSnafu)?; - Ok(NetworkDirectory::new( - &network.name, - &dir.join(ICP_BASE) - .join(CACHE_DIR) - .join("networks") - .join(&network.name), - &self.descriptors, - )) - } - async fn access(&self, network: &Network) -> Result { - match &network.configuration { - Configuration::Managed { managed: _ } => { - let nd = self.get_network_directory(network)?; - Ok(get_managed_network_access(nd).await?) - } - Configuration::Connected { connected: cfg } => { - Ok(get_connected_network_access(cfg, &self.agent).await?) - } - } - } - - async fn urls(&self, network: &Network) -> Result { - match &network.configuration { - Configuration::Managed { managed: _ } => { - let nd = self.get_network_directory(network)?; - Ok(get_managed_network_urls(nd).await?) - } - // A connected network's endpoints are configured, so there is - // nothing to resolve. - Configuration::Connected { connected: cfg } => Ok(NetworkUrls { - api_url: cfg.api_url.clone(), - http_gateway_url: cfg.http_gateway_url.clone(), - }), - } - } - - async fn publish_friendly_domains( - &self, - network: &Network, - collect: &CollectFriendlyDomains<'_>, - ) { - let Configuration::Managed { .. } = &network.configuration else { - return; - }; - let Ok(nd) = self.get_network_directory(network) else { - return; - }; - let Ok(Some(desc)) = nd.load_network_descriptor().await else { - return; - }; - let Some(status_dir) = &desc.status_dir else { - return; - }; - let gateway_url_str = format!("http://{}:{}", desc.gateway.host, desc.gateway.port); - let Ok(gateway_url) = Url::parse(&gateway_url_str) else { - tracing::warn!("Failed to parse gateway URL {gateway_url_str:?} for custom domains"); - return; - }; - let Some(domain) = custom_domains::gateway_domain(&gateway_url) else { - return; - }; +#[cfg(any(test, feature = "test-util"))] +use std::collections::HashMap; - // Only here, past every way this can turn out to have nothing to write, - // is the project asked for any mappings. The descriptor names the - // network the gateway is actually serving, so it — not the - // environment's own view — decides which environments share this - // network and therefore this mapping file. - let env_entries: BTreeMap> = collect(&desc.network) - .into_iter() - .map(|e| (e.environment, e.entries)) - .collect(); - - let extra: Vec<_> = custom_domains::ii_custom_domain_entry(desc.ii, domain) - .into_iter() - .collect(); - if let Err(e) = - custom_domains::write_custom_domains(status_dir, domain, &env_entries, &extra) - { - tracing::warn!("Failed to update custom domains: {e}"); - } - } +/// A [`MockNetworkAccessor`] was asked about a network it was not given. +#[cfg(any(test, feature = "test-util"))] +#[derive(Debug, Snafu)] +#[snafu(display("the {network} network for this project is not running"))] +pub struct NotConfigured { + pub network: String, } -#[cfg(test)] -use std::collections::HashMap; - -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] pub struct MockNetworkAccessor { /// Network-specific access configurations by network name networks: HashMap, } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] impl MockNetworkAccessor { /// Creates a new empty mock network accessor. pub fn new() -> Self { @@ -520,32 +426,22 @@ impl MockNetworkAccessor { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] impl Default for MockNetworkAccessor { fn default() -> Self { Self::new() } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] #[async_trait] impl Access for MockNetworkAccessor { - fn get_network_directory(&self, network: &Network) -> Result { - Ok(NetworkDirectory { - network_name: network.name.clone(), - network_root: PathBuf::new(), - port_descriptor_dir: PathBuf::new(), - }) - } async fn access(&self, network: &Network) -> Result { - self.networks - .get(&network.name) - .cloned() - .ok_or_else(|| AccessError::GetNetworkAccess { - source: GetNetworkAccessError::NetworkNotRunning { - network: network.name.clone(), - }, + self.networks.get(&network.name).cloned().ok_or_else(|| { + AccessError::new(NotConfigured { + network: network.name.clone(), }) + }) } async fn urls(&self, network: &Network) -> Result { diff --git a/crates/icp/src/operations/build.rs b/crates/icp/src/operations/build.rs index 2ef4ecca5..3baeeb8d3 100644 --- a/crates/icp/src/operations/build.rs +++ b/crates/icp/src/operations/build.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use crate::{ Canister, canister::build::{Build, BuildError, Params}, - package::PackageCache, prelude::*, }; use camino_tempfile::tempdir; @@ -46,7 +45,6 @@ pub async fn build( task: &TaskReporter, builder: Arc, artifacts: Arc, - pkg_cache: &PackageCache, ) -> Result<(), BuildOperationError> { let build_dir = tempdir().context(TempDirSnafu)?; let wasm_output_path = build_dir.path().join("out.wasm"); @@ -64,7 +62,6 @@ pub async fn build( environment: environment.to_owned(), }, &reporter, - pkg_cache, ) .await; @@ -95,7 +92,6 @@ pub async fn build_many( environment: &str, builder: Arc, artifacts: Arc, - pkg_cache: &PackageCache, reporter: &Reporter, ) -> Result<(), BuildManyError> { let mut futs = FuturesOrdered::new(); @@ -113,7 +109,6 @@ pub async fn build_many( &task, builder, artifacts, - pkg_cache, ) .await; diff --git a/crates/icp/src/operations/bundle.rs b/crates/icp/src/operations/bundle.rs index 5a9964fea..0edc12d82 100644 --- a/crates/icp/src/operations/bundle.rs +++ b/crates/icp/src/operations/bundle.rs @@ -18,7 +18,6 @@ use crate::{ SyncSteps, load_manifest_from_path, plugin, prebuilt, prebuilt::{LocalSource, SourceField}, }, - package::PackageCache, prelude::*, project::{WorkspaceInstance, WorkspaceInstancesError, workspace_instances}, store_artifact, @@ -192,7 +191,7 @@ pub enum BundleError { #[snafu(display("failed to resolve plugin wasm for canister '{canister}'"))] ResolvePlugin { canister: String, - source: wasm::WasmError, + source: wasm::FetchError, }, #[snafu(display("failed to read plugin wasm for canister '{canister}'"))] @@ -375,7 +374,7 @@ pub async fn create_bundle( environment: &str, builder: Arc, artifacts: Arc, - pkg_cache: &PackageCache, + wasm_fetch: &dyn wasm::Fetch, reporter: &Reporter, output: &Path, ) -> Result<(), BundleError> { @@ -421,7 +420,6 @@ pub async fn create_bundle( environment, builder, artifacts.clone(), - pkg_cache, reporter, ) .await?; @@ -457,7 +455,7 @@ pub async fn create_bundle( instance, &pruned, &*artifacts, - pkg_cache, + wasm_fetch, &mut bundle_artifacts, ) .await?; @@ -676,7 +674,7 @@ async fn prepare_canisters( instance: &Instance, pruned: &Pruned<'_>, artifacts: &dyn store_artifact::Access, - pkg_cache: &PackageCache, + wasm_fetch: &dyn wasm::Fetch, out: &mut BundleArtifacts, ) -> Result>, BundleError> { // Store key -> local name, for rewriting controller references back to the @@ -696,7 +694,7 @@ async fn prepare_canisters( &local_names, pruned, artifacts, - pkg_cache, + wasm_fetch, out, ) .await?; @@ -713,7 +711,7 @@ async fn prepare_canister( local_names: &HashMap<&str, &str>, pruned: &Pruned<'_>, artifacts: &dyn store_artifact::Access, - pkg_cache: &PackageCache, + wasm_fetch: &dyn wasm::Fetch, out: &mut BundleArtifacts, ) -> Result, BundleError> { let local = local_name(&canister.name); @@ -754,7 +752,7 @@ async fn prepare_canister( &path_name, idx, local_names, - pkg_cache, + wasm_fetch, out, ) .await?, @@ -863,22 +861,22 @@ async fn prepare_plugin_step( path_name: &str, idx: usize, local_names: &HashMap<&str, &str>, - pkg_cache: &PackageCache, + wasm_fetch: &dyn wasm::Fetch, out: &mut BundleArtifacts, ) -> Result { let plugin_wasm_path = format!("plugins/{path_name}/{idx}.wasm"); - let resolved = wasm::resolve( - &adapter.source, - canister_path, - adapter.sha256.as_deref(), - &StepReporter::null(), - pkg_cache, - ) - .await - .context(ResolvePluginSnafu { - canister: canister.name.clone(), - })?; + let resolved = wasm_fetch + .wasm( + &adapter.source, + canister_path, + adapter.sha256.as_deref(), + &StepReporter::null(), + ) + .await + .context(ResolvePluginSnafu { + canister: canister.name.clone(), + })?; let plugin_bytes = fs::read(&resolved).context(ReadPluginSnafu { canister: canister.name.clone(), diff --git a/crates/icp/src/operations/deploy.rs b/crates/icp/src/operations/deploy.rs index 9981dc63f..cc560a10b 100644 --- a/crates/icp/src/operations/deploy.rs +++ b/crates/icp/src/operations/deploy.rs @@ -46,7 +46,6 @@ use crate::operations::{ sync::{SyncOperationError, sync_many}, task::{Reporter, Task, TaskReporter, notice}, }; -use crate::package::PackageCache; use crate::project::ArgsField; use crate::{CanisterArgsToBytesError, ProjectLoadError}; @@ -218,7 +217,6 @@ pub struct DeployReport { pub async fn deploy( host: &Host, agent: &LazyAgent<'_>, - pkg_cache: &PackageCache, params: &DeployParams, reporter: &Reporter, report: &mut DeployReport, @@ -240,7 +238,6 @@ pub async fn deploy( environment_selection.name(), host.builder.clone(), host.artifacts.clone(), - pkg_cache, &phase.reporter(), ) .await; @@ -411,7 +408,7 @@ pub async fn deploy( .await; finish(&phase, result)?; - sync(host, pkg_cache, params, agent, reporter).await?; + sync(host, params, agent, reporter).await?; Ok(()) } @@ -522,7 +519,6 @@ async fn create_canisters( /// Run the sync steps of every canister that has any. async fn sync( host: &Host, - pkg_cache: &PackageCache, params: &DeployParams, agent: &Agent, reporter: &Reporter, @@ -619,7 +615,6 @@ async fn sync( urls, canister_ids, proxy, - pkg_cache, &phase.reporter(), ) .await; diff --git a/crates/icp/src/operations/mod.rs b/crates/icp/src/operations/mod.rs index 5d595e661..0cbeb5f22 100644 --- a/crates/icp/src/operations/mod.rs +++ b/crates/icp/src/operations/mod.rs @@ -2,7 +2,6 @@ pub mod binding_env_vars; pub mod build; pub mod bundle; pub mod candid_compat; -pub mod canister_migration; pub mod create; pub mod deploy; pub mod install; @@ -10,10 +9,8 @@ pub mod proxy; pub mod proxy_management; pub mod recover_cycles; pub mod settings; -pub mod snapshot_transfer; pub mod sync; pub mod task; -pub mod token; pub mod misc; pub mod wasm; diff --git a/crates/icp/src/operations/sync.rs b/crates/icp/src/operations/sync.rs index 1dab5605b..3976615f9 100644 --- a/crates/icp/src/operations/sync.rs +++ b/crates/icp/src/operations/sync.rs @@ -2,7 +2,6 @@ use crate::{ Canister, canister::sync::{Params, Synchronize, SynchronizeError}, network::NetworkUrls, - package::PackageCache, prelude::{Path, PathBuf}, }; use candid::Principal; @@ -37,7 +36,6 @@ async fn sync_canister( canister_ids: &BTreeMap, proxy: Option, task: &TaskReporter, - pkg_cache: &PackageCache, ) -> Result, SynchronizeError> { let step_count = canister_info.sync.steps.len(); let mut stderr_lines = Vec::new(); @@ -61,7 +59,6 @@ async fn sync_canister( }, agent, &reporter, - pkg_cache, ) .await; @@ -98,7 +95,6 @@ pub async fn sync_many( urls: NetworkUrls, canister_ids: BTreeMap, proxy: Option, - pkg_cache: &PackageCache, reporter: &Reporter, ) -> Result<(), SyncOperationError> { let mut futs = FuturesOrdered::new(); @@ -129,7 +125,6 @@ pub async fn sync_many( &canister_ids, proxy, &task, - pkg_cache, ) .await; diff --git a/crates/icp/src/prelude.rs b/crates/icp/src/prelude.rs index 3cf6e17e0..ff5c9f62c 100644 --- a/crates/icp/src/prelude.rs +++ b/crates/icp/src/prelude.rs @@ -11,3 +11,8 @@ pub const IC_MAINNET_NETWORK_GATEWAY_URL: &str = "https://icp.net"; pub const IC: &str = "ic"; /// Name of the implicit local managed network and its implicit environment pub const LOCAL: &str = "local"; + +/// The IC mainnet root key, as served by the NNS. Pinned rather than fetched: +/// a key fetched over the network is only as trustworthy as the response that +/// carried it. +pub const IC_ROOT_KEY: &[u8; 133] = b"\x30\x81\x82\x30\x1d\x06\x0d\x2b\x06\x01\x04\x01\x82\xdc\x7c\x05\x03\x01\x02\x01\x06\x0c\x2b\x06\x01\x04\x01\x82\xdc\x7c\x05\x03\x02\x01\x03\x61\x00\x81\x4c\x0e\x6e\xc7\x1f\xab\x58\x3b\x08\xbd\x81\x37\x3c\x25\x5c\x3c\x37\x1b\x2e\x84\x86\x3c\x98\xa4\xf1\xe0\x8b\x74\x23\x5d\x14\xfb\x5d\x9c\x0c\xd5\x46\xd9\x68\x5f\x91\x3a\x0c\x0b\x2c\xc5\x34\x15\x83\xbf\x4b\x43\x92\xe4\x67\xdb\x96\xd6\x5b\x9b\xb4\xcb\x71\x71\x12\xf8\x47\x2e\x0d\x5a\x4d\x14\x50\x5f\xfd\x74\x84\xb0\x12\x91\x09\x1c\x5f\x87\xb9\x88\x83\x46\x3f\x98\x09\x1a\x0b\xaa\xae"; diff --git a/crates/icp/src/project.rs b/crates/icp/src/project.rs index 057e41540..8eb20156b 100644 --- a/crates/icp/src/project.rs +++ b/crates/icp/src/project.rs @@ -455,13 +455,12 @@ async fn build_manifest_canisters( // The template rendered, so an unpinned download is now known // good and safe to cache. Committing only here is what keeps a // bad remote response from becoming sticky. - if let Some(pending) = fetched.pending_cache { - recipe_resolver - .commit(pending) - .await - .context(CacheRecipeSnafu { + if fetched.deferred { + recipe_resolver.commit(recipe, &fetched).await.context( + CacheRecipeSnafu { recipe_type: recipe.recipe_type.clone(), - })?; + }, + )?; } // The manifest's own sync steps run after the recipe's. @@ -1662,7 +1661,7 @@ mod recipe_sync_tests { async fn resolve(&self, _recipe: &Recipe) -> Result { Ok(Fetched { template: self.0.to_owned(), - pending_cache: None, + deferred: false, }) } } diff --git a/crates/icp/src/store_artifact.rs b/crates/icp/src/store_artifact.rs index 913a59de7..e5bc6fc40 100644 --- a/crates/icp/src/store_artifact.rs +++ b/crates/icp/src/store_artifact.rs @@ -1,5 +1,5 @@ use std::sync::Arc; -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] use std::{collections::HashMap, sync::Mutex}; use crate::{ @@ -57,11 +57,11 @@ pub enum LookupArtifactError { LockError { source: crate::fs::lock::LockError }, } -pub(crate) struct ArtifactStore { +pub struct ArtifactStore { project_root_locate: Arc, } -struct ArtifactPaths { +pub struct ArtifactPaths { dir: PathBuf, } @@ -110,7 +110,7 @@ impl PathsAccess for ArtifactPaths { } impl ArtifactStore { - pub(crate) fn new(project_root_locate: Arc) -> Self { + pub fn new(project_root_locate: Arc) -> Self { Self { project_root_locate, } @@ -177,30 +177,30 @@ impl Access for ArtifactStore { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] /// In-memory mock implementation of `Access`. pub(crate) struct MockInMemoryArtifactStore { store: Mutex>>, } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] impl MockInMemoryArtifactStore { /// Creates a new empty in-memory artifact store. - pub(crate) fn new() -> Self { + pub fn new() -> Self { Self { store: Mutex::new(HashMap::new()), } } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] impl Default for MockInMemoryArtifactStore { fn default() -> Self { Self::new() } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] #[async_trait] impl Access for MockInMemoryArtifactStore { async fn save(&self, name: &str, wasm: &[u8]) -> Result<(), SaveError> { diff --git a/crates/icp/src/store_id.rs b/crates/icp/src/store_id.rs index 872cd8e42..acf74f888 100644 --- a/crates/icp/src/store_id.rs +++ b/crates/icp/src/store_id.rs @@ -133,13 +133,13 @@ pub enum CleanupError { /// Store of canister ID mappings for environments. /// /// Each environment has a separate file storing its canister IDs mapping. -pub(crate) struct AccessImpl { +pub struct AccessImpl { project_root_locate: Arc, lock: Mutex<()>, } impl AccessImpl { - pub(crate) fn new(project_root_locate: Arc) -> Self { + pub fn new(project_root_locate: Arc) -> Self { Self { project_root_locate, lock: Mutex::new(()), @@ -272,22 +272,22 @@ impl AccessImpl { } } -#[cfg(test)] -pub(crate) mod mock { +#[cfg(any(test, feature = "test-util"))] +pub mod mock { use super::*; /// In-memory mock implementation of `Access`. /// /// There are two separate stores for cache and data, to allow testing both paths. /// Each store keys on the environment name. /// The value is a mapping from canister names to their principals. - pub(crate) struct MockInMemoryIdStore { + pub struct MockInMemoryIdStore { cache: Mutex>, data: Mutex>, } impl MockInMemoryIdStore { /// Creates a new empty in-memory ID store. - pub(crate) fn new() -> Self { + pub fn new() -> Self { Self { cache: Mutex::new(BTreeMap::new()), data: Mutex::new(BTreeMap::new()),